Overview

This notebook evaluates the behavioural predictions (RT, accuracy) that follow from the predicted rates of forgetting. We simulate the behavioural predictions that the adaptive fact learning system would have made, if it had used the predicted rate of forgetting as the starting estimate in the learning sequence.

Setup

library(fst)
library(data.table)
library(tidyr)
Registered S3 methods overwritten by 'tibble':
  method     from  
  format.tbl pillar
  print.tbl  pillar
library(purrr)

Attaching package: 'purrr'
The following object is masked from 'package:data.table':

    transpose
library(furrr)
Loading required package: future
library(stringr)
library(ggplot2)
library(patchwork)
library(wesanderson)
library(lme4)
Loading required package: Matrix

Attaching package: 'Matrix'
The following objects are masked from 'package:tidyr':

    expand, pack, unpack
library(lmerTest)

Attaching package: 'lmerTest'
The following object is masked from 'package:lme4':

    lmer
The following object is masked from 'package:stats':

    step
library(multcomp)
Loading required package: mvtnorm
Loading required package: survival

Attaching package: 'survival'
The following object is masked from 'package:future':

    cluster
Loading required package: TH.data
Loading required package: MASS

Attaching package: 'MASS'
The following object is masked from 'package:patchwork':

    area

Attaching package: 'TH.data'
The following object is masked from 'package:MASS':

    geyser
source(file.path("..", "scripts", "99_slimstampen_model_funs.R"))

Attaching package: 'dplyr'
The following object is masked from 'package:MASS':

    select
The following objects are masked from 'package:data.table':

    between, first, last
The following objects are masked from 'package:stats':

    filter, lag
The following objects are masked from 'package:base':

    intersect, setdiff, setequal, union
future::plan("multiprocess", workers = 6) # Set to desired number of cores
theme_set(theme_light(base_size = 14) +
            theme(strip.text = element_text(colour = "black")))

condition_colours <- wes_palette("Darjeeling1", n = 5)
condition_colours[c(2, 4, 5)] <- condition_colours[c(4, 5, 2)]

dataset_colours <- wes_palette("Darjeeling2", n = 5)[c(2, 3)]

Helper functions

load_data_with_predictions <- function (course) {
  
  # Data
  d_full <- read_fst(file.path("..", "data", paste0("formatted_", str_replace_all(course, " ", "_"), ".fst")))
  setDT(d_full)
  d <- d_full[!is.na(fact_id), .(user_id, fact_id, start_time, rt, correct)]
  rm(d_full)
  gc()
  setorder(d, user_id, fact_id, start_time)
  
  # ROF predictions
  pred_user <- read_fst(file.path("..", "data", "predictions", paste0("pred_v_obs_user_", str_replace_all(course, " ", "_"), ".fst")))
  pred_fact <- read_fst(file.path("..", "data", "predictions", paste0("pred_v_obs_fact_", str_replace_all(course, " ", "_"), ".fst")))
  pred_fact_user <- read_fst(file.path("..", "data", "predictions", paste0("pred_fact_and_user_", str_replace_all(course, " ", "_"), ".fst")))
  setDT(pred_user)
  setDT(pred_fact)
  setDT(pred_fact_user)
  
  # Remove NA and duplicates
  pred_user <- unique(pred_user[!is.na(alpha)])
  pred_fact <- unique(pred_fact[!is.na(alpha)])
  pred_fact_user <- unique(pred_fact_user[!is.na(alpha)])
  
  # Make Domain prediction
  pred_domain <- mean(unique(pred_fact, by = c("fact_id"))$pred_fact)
  pred_default <- 0.3
  
  # Combine
  setnames(pred_user, "n_train_obs", "n_train_obs_user")
  setnames(pred_fact, "n_train_obs", "n_train_obs_fact")
  pred_all <- merge(pred_user, pred_fact, by = c("user_id", "fact_id", "alpha", "n_reps"), all = TRUE)
  pred_all <- merge(pred_all, pred_fact_user, by = c("user_id", "fact_id", "alpha"), all = TRUE)
  pred_all[, pred_default := pred_default]
  pred_all[, pred_domain := pred_domain]

  d_prep <- d[, .(user_id,
                  fact_id,
                  text = "",
                  start_time,
                  rt,
                  correct,
                  threshold = -0.8)]
  
  d_pred <- merge(pred_all, d_prep, by = c("user_id", "fact_id"))
  
  return(d_pred)
}

predict_behaviour <- function (d) {
  
    # Process the data in manageable chunks
    chunk_size <- 1e4
    obs <- unique(d[, .(user_id, fact_id)])
    obs_chunks <- c(seq(1, nrow(obs), by = chunk_size), nrow(obs)+1)
    
    pred_beh <- map_dfr(1:(length(obs_chunks)-1), function (i) {
      
      msg <- paste("Chunk", i, "/", length(obs_chunks)-1)
      system(paste("echo", msg))
      
      obs_i <- obs[obs_chunks[i]:obs_chunks[i+1]-1]
      d_i <- d[obs_i, on = .(user_id, fact_id)]
      d_i_list <- split(d_i, by = c("user_id", "fact_id"), drop = TRUE)
      
      pred_beh_i <- future_map_dfr(d_i_list, function (learn_seq) {
        
        # Organise predictions for this sequence
        learn_seq_preds <- pivot_longer(
          learn_seq[1,], 
          pred_user:pred_domain,
          names_to = "prediction_type",
          names_prefix = "pred_",
          values_to = "predicted_alpha"
        )
        setDT(learn_seq_preds)
        
        # Look only at trial 3 in each sequence
        trial <- learn_seq[3,]
        delay <- learn_seq[2:3, diff(start_time)]
        
        # Calculate behavioural predictions for each prediction method
        map_dfr(seq(nrow(learn_seq_preds)), function (j) {
          
          prediction_type <- learn_seq_preds[j, prediction_type]
          predicted_alpha <- learn_seq_preds[j, predicted_alpha]
    
          predicted_activation <- NA
          predicted_accuracy <- NA
          predicted_rt <- NA
    
          if (!is.na(predicted_alpha)) {
            
            predicted_activation <- calculate_activation(
              time = trial[, start_time],
              id = trial[, fact_id],
              factalpha = predicted_alpha,
              responses = learn_seq[1:2,]
            )
      
            predicted_accuracy <- p_recall(
              activation = predicted_activation,
              threshold = -0.8,
              activation_noise = 0.5
            )
            
            predicted_rt <- estimate_reaction_time_from_activation(
              activation = predicted_activation,
              reading_time = 300
            )
          }
          
          return(
            list(
              user_id = trial[, user_id],
              fact_id = trial[, fact_id],
              delay = delay,
              correct = trial[, correct],
              rt = trial[, rt],
              prediction_type = prediction_type,
              predicted_alpha = predicted_alpha,
              predicted_activation = predicted_activation,
              predicted_accuracy = predicted_accuracy,
              predicted_rt = predicted_rt
            )
          )
          
        })
      })
    })
    
    setDT(pred_beh)
    
    # Remove rows without prediction
    pred_beh <- pred_beh[!is.na(predicted_alpha)]
    pred_beh <- pred_beh[!is.infinite(predicted_rt)]
    
    # Set proper condition labels
    condition_labels <- data.table(
      prediction_type = c("default", "domain", "fact", "user", "fact_user"),
      prediction_label = factor(
        c("Default", "Domain", "Fact", "Learner", "Fact & Learner"),
        levels = c("Default", "Domain", "Fact", "Learner", "Fact & Learner")
      )
    )
    pred_beh <- pred_beh[condition_labels, on = .(prediction_type)]
    
    
    return(pred_beh)
}

Calculate predictions

Load test set data with rate of forgetting predictions:

pred_gl <- load_data_with_predictions("Grandes Lignes")
pred_ss <- load_data_with_predictions("Stepping Stones")

Calculate behavioural predictions for trial 3:

pred_gl_beh_path <- file.path("..", "data", "predictions", "pred_behaviour_gl.fst")
pred_ss_beh_path <- file.path("..", "data", "predictions", "pred_behaviour_ss.fst")

if (!file.exists(pred_gl_beh_path)) {
  pred_gl_beh <- predict_behaviour(pred_gl)
  write_fst(pred_gl_beh, pred_gl_beh_path)
} else {
  pred_gl_beh <- read_fst(pred_gl_beh_path)
  setDT(pred_gl_beh)
}

if (!file.exists(pred_ss_beh_path)) {
  pred_ss_beh <- predict_behaviour(pred_ss)
  write_fst(pred_ss_beh, pred_ss_beh_path)
} else {
  pred_ss_beh <- read_fst(pred_ss_beh_path)
  setDT(pred_ss_beh)
}
pred_beh <- rbind(pred_gl_beh[, course := "French"],
                  pred_ss_beh[, course := "English"])

rm(pred_gl_beh, pred_ss_beh)
gc()
            used   (Mb) gc trigger    (Mb)   max used    (Mb)
Ncells   2213155  118.2    3995827   213.5    2500112   133.6
Vcells 524097644 3998.6 1371488967 10463.7 1369780600 10450.6

Activation

p_act_dist <- ggplot(pred_beh[between(predicted_activation, -2, 0)], aes(x = predicted_activation, fill = prediction_label)) +
  facet_grid(course ~ prediction_label, scales = "free_y") +
  geom_histogram(aes(y = ..density..), binwidth = .01) +
  guides(fill = "none") +
  labs(x = "RT (in s)",
       y = "Density") +
  scale_fill_manual(values = condition_colours)

# p_act_dist

ggsave(plot = p_act_dist, file.path("..", "output", "activation_distribution.png"),
       device = "png", width = 7.5, height = 4.5)

rm(p_act_dist)

Response time

Distribution of observed correct RT (truncated at 25 seconds for readability):

p_rt_dist <- ggplot(pred_beh[correct == 1 & between(rt, 0, 25000)], aes(x = rt/1000)) +
  facet_grid(course ~ ., scales = "free_y") +
  geom_histogram(aes(y = ..density..), binwidth = .1) +
  guides(fill = "none") +
  labs(x = "RT (in s)",
       y = "Density")

# p_rt_dist

ggsave(plot = p_rt_dist, file.path("..", "output", "rt_observed_distribution.png"),
       device = "png", width = 6, height = 6)

rm(p_rt_dist)

Predicted response time

Distribution of predictions

p_rt_pred_dist <- ggplot(pred_beh, aes(x = predicted_rt/1000)) +
  facet_grid(course ~ ., scales = "free_y") +
  geom_histogram(aes(y = ..density..), binwidth = .1) +
  guides(fill = "none") +
  labs(x = "Predicted RT (in s)",
       y = "Density")

# p_rt_pred_dist

ggsave(plot = p_rt_pred_dist, file.path("..", "output", "rt_predicted_distribution.png"),
       device = "png", width = 6, height = 6)

rm(p_rt_pred_dist)

Predicted vs observed values

Calculate absolute prediction error:

pred_rt_error <- (pred_beh
                  [correct == 1]
                  [between(rt, 0, 25000)]
                  [, rt_pred_error := predicted_rt - rt]
                  [, abs_rt_pred_error := abs(rt_pred_error)]
)

pred_rt_error_avg <- pred_rt_error[, .(mae = mean(abs_rt_pred_error),
                                       ae_se = sd(abs_rt_pred_error)/.N), 
                                   by = .(course, prediction_label)]
  
n_obs <- pred_rt_error[, .N, by = .(course, prediction_label)]
# plot_range <- range(pred_beh$predicted_rt/1000, na.rm = TRUE)
plot_range <- c(0, 10)
plot_breaks <- seq(0, 10, by = 2)
# 
# plot_range <- quantile(pred_beh$predicted_rt/1000, c(.005, .995))

p_rt_pred_v_obs <- ggplot(pred_beh[correct == 1], aes(x = predicted_rt/1000, y = rt/1000, colour = prediction_label)) +
  facet_grid(course ~ prediction_label) +
  geom_abline(slope = 1, intercept = 0, lty = 3, alpha = 0.75) +
  geom_point(alpha = .1, size = .1, pch = ".") +
  geom_smooth(method = "lm", formula = y ~ x, colour = "black") +
  geom_label(data = pred_rt_error_avg,
            aes(label = paste("MAE =", formatC(mae/1000, digits = 3, flag = "#"))),
            x = plot_range[2], y = plot_range[1],
            hjust = 1, colour = "NA", size = 3,
            alpha = .9,
            label.size = NA) +
  geom_text(data = pred_rt_error_avg,
            aes(label = paste("MAE =", formatC(mae/1000, digits = 3, flag = "#"))),
            x = plot_range[2], y = plot_range[1],
            hjust = 1, colour = "black", size = 3) +
  geom_label(data = n_obs,
            aes(label = paste("n =", scales::comma(N))),
            x = plot_range[2],
            y = plot_range[2],
            hjust = 1, colour = "NA", size = 3,
            alpha = .9,
            label.size = NA) +
  geom_text(data = n_obs,
            aes(label = paste("n =", scales::comma(N))),
            x = plot_range[2],
            y = plot_range[2],
            hjust = 1, colour = "black", size = 3) +
  guides(colour = "none") +
  labs(x = "Predicted RT (s)",
       y = "Observed RT (s)") +
  coord_fixed(ratio = 1, xlim = plot_range, ylim = plot_range) +
  scale_x_continuous(breaks = plot_breaks) +
  scale_y_continuous(breaks = plot_breaks) +
  scale_colour_manual(values = condition_colours)

p_rt_pred_v_obs

ggsave(plot = p_rt_pred_v_obs, file.path("..", "output", "rt_predicted_vs_observed.png"),
       device = "png", width = 10, height = 4.5)

rm(p_rt_pred_v_obs)

Prediction error

Distribution of prediction error (truncated to [-5, 5] for readability):

p_rt_pred_error <- ggplot(pred_rt_error, aes(x = rt_pred_error/1000, fill = prediction_label)) +
  facet_grid(prediction_label ~ course , scales = "free_y") +
  geom_histogram(aes(y = ..density..), binwidth = .1) +
  guides(fill = "none") +
  labs(x = "RT prediction error in s (predicted - observed)",
       y = "Density") +
  coord_cartesian(xlim = c(-5, 5)) +
  scale_fill_manual(values = condition_colours)

p_rt_pred_error

ggsave(plot = p_rt_pred_error, file.path("..", "output", "rt_prediction_error.png"),
       device = "png", width = 5, height = 7.5)

rm(p_rt_pred_error)

Absolute prediction error

p_rt_abs_pred_error <- ggplot(pred_rt_error, aes(x = abs_rt_pred_error/1000, fill = prediction_label)) +
  facet_grid(prediction_label ~ course, scales = "free_y") +
  geom_histogram(aes(y = ..density..), binwidth = .1) +
  guides(fill = "none") +
  labs(x = "RT prediction error in s (predicted - observed)",
       y = "Density") +
  coord_cartesian(xlim = c(0, 5)) +
  scale_fill_manual(values = condition_colours)

p_rt_abs_pred_error

ggsave(plot = p_rt_abs_pred_error, file.path("..", "output", "rt_absolute_prediction_error.png"),
       device = "png", width = 5, height = 7.5)

rm(p_rt_abs_pred_error)
ggplot(pred_rt_error_avg, aes(x = prediction_label, y = mae/1000, colour = course)) +
  geom_boxplot(data = pred_rt_error,
               aes(y = abs_rt_pred_error/1000, group = interaction(course, prediction_label)),
               colour = "grey70",
               width = .25,
               outlier.shape = NA,
               position = position_dodge(width = .5)) +
  geom_errorbar(aes(ymin = mae/1000 - ae_se/1000, ymax = mae/1000 + ae_se/1000), width = 0, position = position_dodge(width = .5)) +
  geom_point(position = position_dodge(width = .5)) +
  coord_cartesian(ylim = c(0, 2.5)) +
  labs(x = "Method",
       y = "Absolute RT prediction error (in s)",
       colour = "Course")

Fit a regression model on absolute RT prediction error.

French
m_rt_pred_error_gl_file <- file.path("..", "data", "model_fits", "m_rt_pred_error_Grandes_Lignes.rda")

if (file.exists(m_rt_pred_error_gl_file)) {
  load(m_rt_pred_error_gl_file)
} else {
  
  pred_gl_reg <- (
    pred_rt_error
    [course == "French"]
    [sample(.N, 1e6)]
    [, .(prediction_label, abs_rt_pred_error, user_id, fact_id)]
  )
  
  m_rt_pred_error_gl <- lmer(abs_rt_pred_error ~ prediction_label + 
                               (1 | user_id) + (1 | fact_id),
                             data = pred_gl_reg,
                             control = lmerControl(optimizer ="bobyqa"))
  
  save(m_rt_pred_error_gl, file = m_rt_pred_error_gl_file)
}

summary(m_rt_pred_error_gl)
Linear mixed model fit by REML. t-tests use Satterthwaite's method [
lmerModLmerTest]
Formula: 
abs_rt_pred_error ~ prediction_label + (1 | user_id) + (1 | fact_id)
   Data: pred_gl_reg
Control: lmerControl(optimizer = "bobyqa")

REML criterion at convergence: 18047298

Scaled residuals: 
    Min      1Q  Median      3Q     Max 
-2.5110 -0.4053 -0.1761  0.0625 11.1385 

Random effects:
 Groups   Name        Variance Std.Dev.
 user_id  (Intercept)  220053   469.1  
 fact_id  (Intercept)  194518   441.0  
 Residual             3843235  1960.4  
Number of obs: 1000000, groups:  user_id, 40820; fact_id, 22762

Fixed effects:
                                 Estimate Std. Error         df t value
(Intercept)                      1433.560      6.509  53722.081 220.257
prediction_labelDomain            -19.468      6.192 974447.696  -3.144
prediction_labelFact             -118.481      6.218 974827.243 -19.056
prediction_labelLearner           -42.334      6.270 975733.404  -6.751
prediction_labelFact & Learner   -101.382      6.300 976077.378 -16.091
                               Pr(>|t|)    
(Intercept)                     < 2e-16 ***
prediction_labelDomain          0.00167 ** 
prediction_labelFact            < 2e-16 ***
prediction_labelLearner        1.46e-11 ***
prediction_labelFact & Learner  < 2e-16 ***
---
Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1

Correlation of Fixed Effects:
            (Intr) prdc_D prdc_F prdc_L
prdctn_lblD -0.474                     
prdctn_lblF -0.468  0.497              
prdctn_lblL -0.462  0.492  0.490       
prdctn_lF&L -0.455  0.490  0.489  0.486

Compare different prediction types to each other:

ht_rt_gl <- glht(m_rt_pred_error_gl, linfct = mcp(prediction_label = "Tukey"))
summary(ht_rt_gl)

     Simultaneous Tests for General Linear Hypotheses

Multiple Comparisons of Means: Tukey Contrasts


Fit: lmer(formula = abs_rt_pred_error ~ prediction_label + (1 | user_id) + 
    (1 | fact_id), data = pred_gl_reg, control = lmerControl(optimizer = "bobyqa"))

Linear Hypotheses:
                              Estimate Std. Error z value Pr(>|z|)    
Domain - Default == 0          -19.468      6.192  -3.144  0.01420 *  
Fact - Default == 0           -118.481      6.218 -19.056  < 0.001 ***
Learner - Default == 0         -42.334      6.270  -6.751  < 0.001 ***
Fact & Learner - Default == 0 -101.382      6.300 -16.091  < 0.001 ***
Fact - Domain == 0             -99.013      6.225 -15.905  < 0.001 ***
Learner - Domain == 0          -22.867      6.278  -3.642  0.00249 ** 
Fact & Learner - Domain == 0   -81.914      6.308 -12.986  < 0.001 ***
Learner - Fact == 0             76.146      6.304  12.079  < 0.001 ***
Fact & Learner - Fact == 0      17.099      6.329   2.702  0.05360 .  
Fact & Learner - Learner == 0  -59.047      6.374  -9.264  < 0.001 ***
---
Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
(Adjusted p values reported -- single-step method)

Inspect the model's residuals:

qqnorm(resid(m_rt_pred_error_gl))
qqline(resid(m_rt_pred_error_gl), col = "red")

plot(m_rt_pred_error_gl)

English
m_rt_pred_error_ss_file <- file.path("..", "data", "model_fits", "m_rt_pred_error_Stepping_Stones.rda")

if (file.exists(m_rt_pred_error_ss_file)) {
  load(m_rt_pred_error_ss_file)
} else {
  
  pred_ss_reg <- (
    pred_rt_error
    [course == "English"]
    [sample(.N, 1e6)]
    [, .(prediction_label, abs_rt_pred_error, user_id, fact_id)]
  )
  
  m_rt_pred_error_ss <- lmer(abs_rt_pred_error ~ prediction_label + 
                               (1 | user_id) + (1 | fact_id),
                             data = pred_ss_reg,
                             control = lmerControl(optimizer ="bobyqa")
  )
  
  save(m_rt_pred_error_ss, file = m_rt_pred_error_ss_file)
}

summary(m_rt_pred_error_ss)
Linear mixed model fit by REML. t-tests use Satterthwaite's method [
lmerModLmerTest]
Formula: 
abs_rt_pred_error ~ prediction_label + (1 | user_id) + (1 | fact_id)
   Data: pred_ss_reg
Control: lmerControl(optimizer = "bobyqa")

REML criterion at convergence: 18097553

Scaled residuals: 
    Min      1Q  Median      3Q     Max 
-1.6170 -0.3973 -0.1997  0.0215 10.9768 

Random effects:
 Groups   Name        Variance Std.Dev.
 user_id  (Intercept)  150645   388.1  
 fact_id  (Intercept)   72213   268.7  
 Residual             4075141  2018.7  
Number of obs: 1000000, groups:  user_id, 85884; fact_id, 45600

Fixed effects:
                                 Estimate Std. Error         df t value
(Intercept)                      1312.875      5.205 161344.300 252.225
prediction_labelDomain            -27.214      6.416 984927.056  -4.242
prediction_labelFact              -58.734      6.448 985192.856  -9.109
prediction_labelLearner           -41.073      6.467 984764.807  -6.351
prediction_labelFact & Learner    -64.932      6.493 985035.484 -10.000
                               Pr(>|t|)    
(Intercept)                     < 2e-16 ***
prediction_labelDomain         2.22e-05 ***
prediction_labelFact            < 2e-16 ***
prediction_labelLearner        2.14e-10 ***
prediction_labelFact & Learner  < 2e-16 ***
---
Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1

Correlation of Fixed Effects:
            (Intr) prdc_D prdc_F prdc_L
prdctn_lblD -0.617                     
prdctn_lblF -0.613  0.498              
prdctn_lblL -0.609  0.496  0.494       
prdctn_lF&L -0.606  0.494  0.492  0.491

Compare different prediction types to each other:

ht_rt_ss <- glht(m_rt_pred_error_ss, linfct = mcp(prediction_label = "Tukey"))
summary(ht_rt_ss)

     Simultaneous Tests for General Linear Hypotheses

Multiple Comparisons of Means: Tukey Contrasts


Fit: lmer(formula = abs_rt_pred_error ~ prediction_label + (1 | user_id) + 
    (1 | fact_id), data = pred_ss_reg, control = lmerControl(optimizer = "bobyqa"))

Linear Hypotheses:
                              Estimate Std. Error z value Pr(>|z|)    
Domain - Default == 0          -27.214      6.416  -4.242  < 0.001 ***
Fact - Default == 0            -58.734      6.448  -9.109  < 0.001 ***
Learner - Default == 0         -41.073      6.467  -6.351  < 0.001 ***
Fact & Learner - Default == 0  -64.932      6.493 -10.000  < 0.001 ***
Fact - Domain == 0             -31.520      6.445  -4.890  < 0.001 ***
Learner - Domain == 0          -13.859      6.466  -2.143  0.20185    
Fact & Learner - Domain == 0   -37.717      6.491  -5.811  < 0.001 ***
Learner - Fact == 0             17.661      6.497   2.718  0.05136 .  
Fact & Learner - Fact == 0      -6.198      6.521  -0.950  0.87700    
Fact & Learner - Learner == 0  -23.859      6.540  -3.648  0.00242 ** 
---
Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
(Adjusted p values reported -- single-step method)

Inspect the model's residuals:

qqnorm(resid(m_rt_pred_error_ss))
qqline(resid(m_rt_pred_error_ss), col = "red")

plot(m_rt_pred_error_ss)

Comparison
ht_rt_gl_tidy <- broom::tidy(confint(ht_rt_gl))
ht_rt_ss_tidy <- broom::tidy(confint(ht_rt_ss))
setDT(ht_rt_gl_tidy)
setDT(ht_rt_ss_tidy)

ht_rt_both_tidy <- rbind(ht_rt_gl_tidy[, course := "French"],
                      ht_rt_ss_tidy[, course := "English"])
p_rt_pred_error_comp <- ggplot(ht_rt_both_tidy, aes(x = lhs, y = estimate, ymin = conf.low, ymax = conf.high, colour = course)) +
  geom_hline(yintercept = 0, linetype = "11", colour = "grey60") +
  geom_errorbar(width = 0.1) + 
  geom_point() +
  labs(x = "Linear hypotheses",
       y = "Estimate",
       caption = "Tukey's range test. Error bars show 95% family-wise confidence level.",
       colour = "Course") +
  coord_flip()

p_rt_pred_error_comp

ggsave(plot = p_rt_pred_error_comp, file.path("..", "output", "rt_prediction_error_comparisons.png"),
       device = "png", width = 7.5, height = 5)

rm(p_rt_pred_error_comp)
Summary plot
pred_rt_error_avg[, prediction_rank := frank(-mae), by = .(course)]
annotation_df_ss <- data.table(
  course = rep("English", 10),
  start = c(1, 1, 1, 1,
            2, 2, 2,
            3, 3,
            4
  ),
  end = c(2, 3, 4, 5,
          3, 4, 5,
          4, 5,
          5
  ),
  y = seq(max(pred_rt_error_avg$mae)*1.01 + 45, max(pred_rt_error_avg$mae)*1.01, by = -5),
  label = c("p < .001", "p < .001", "p < .001", "p < .001",
            "n.s.", "p < .001", "p < .001",
            "n.s.", "p < .01",
            "n.s.")
)

annotation_df_gl <- data.table(
  course = rep("French", 10),
  start = c(1, 1, 1, 1,
            2, 2, 2,
            3, 3,
            4
  ),
  end = c(2, 3, 4, 5,
          3, 4, 5,
          4, 5,
          5
  ),
  y = seq(max(pred_rt_error_avg$mae)*1.01 + 45, max(pred_rt_error_avg$mae)*1.01, by = -5),
  label = c("p < .05", "p < .001", "p < .001", "p < .001",
            "p < .01", "p < .001", "p < .001",
            "p < .001", "p < .001",
            "n.s.")
)

annotation_df_rt <- rbind(annotation_df_ss, annotation_df_gl)
annotation_df_rt[, label := factor(label, levels = c("p < .001", "p < .01", "p < .05", "n.s."))]
p_rt_pred_error_summary <- ggplot(pred_rt_error_avg, aes(x = prediction_rank, y = mae)) +
  facet_grid(~ course) +
  geom_line(data = annotation_df_rt,
            aes(x = 1, y = 1300, lty = label, alpha = label, colour = NULL)) + # Dummy line to get legend
  geom_line(aes(colour = course, group = course)) +
  geom_errorbar(aes(ymin = mae - ae_se, ymax = mae + ae_se), width = 0) +
  geom_point(aes(colour = course, group = course)) +
  geom_label(aes(label = prediction_label), 
             colour = "black", 
             alpha = .9,
             label.size = NA, 
             nudge_y = -15) +
  labs(x = NULL,
       y = "Absolute prediction error:\nresponse time (s)",
       colour = "Course") +
  scale_x_continuous(expand = expansion(add = .75), breaks = NULL) +
  scale_y_continuous(labels = scales::comma_format(big.mark = ".")) +
  scale_colour_manual(values = dataset_colours) +
  scale_linetype_manual(values = c("p < .001" = 1,
                                   "p < .01" = 5,
                                   "p < .05" = 2,
                                   "n.s." = 3),
                        name = "Pairwise comparison:") +
  scale_alpha_manual(values = c("p < .001" = 1,
                                "p < .01" = .75,
                                "p < .05" = .5, 
                                "n.s." = .25), 
                     name = "Pairwise comparison:") +
  guides(colour = "none") +
  ggsignif::geom_signif(data = annotation_df_rt,
                        aes(xmin = start, xmax = end, annotations = "", 
                            y_position = y, lty = label, alpha = label),
                        tip_length = 0,
                        manual = TRUE)  +
  theme(legend.position = "bottom",
        legend.justification = "right")
Warning: Ignoring unknown aesthetics: xmin, xmax, annotations, y_position
p_rt_pred_error_summary
Warning in prettyNum(.Internal(format(x, trim, digits, nsmall, width, 3L, :
'big.mark' and 'decimal.mark' are both '.', which could be confusing
Warning in prettyNum(.Internal(format(x, trim, digits, nsmall, width, 3L, :
'big.mark' and 'decimal.mark' are both '.', which could be confusing

ggsave(file.path("..", "output", "rt_absolute_prediction_error_summary.png"),
       device = "png", width = 10, height = 4)
Warning in prettyNum(.Internal(format(x, trim, digits, nsmall, width, 3L, :
'big.mark' and 'decimal.mark' are both '.', which could be confusing

Warning in prettyNum(.Internal(format(x, trim, digits, nsmall, width, 3L, :
'big.mark' and 'decimal.mark' are both '.', which could be confusing
Improvement

How big was the improvement from worst to best prediction method?

French:

# Absolute change
ht_rt_gl_tidy[lhs == "Fact - Default", estimate[1]]
[1] -118.4807
# % change
scales::percent(
  ht_rt_gl_tidy[lhs == "Fact - Default", estimate[1]] / fixef(m_rt_pred_error_gl)[[1]],
  accuracy = .1)
[1] "-8.3%"
-8.3%

English:

# Absolute change
ht_rt_ss_tidy[lhs == "Fact & Learner - Default", estimate[1]]
[1] -64.93171
# % change
scales::percent(
  ht_rt_ss_tidy[lhs == "Fact & Learner - Default", estimate[1]] / fixef(m_rt_pred_error_ss)[[1]],
  accuracy = .1)
[1] "-4.9%"
-4.9%

Response accuracy

Predicted response accuracy

Distribution of predictions

p_acc_pred_dist <- ggplot(pred_beh, aes(x = predicted_accuracy, fill = prediction_label)) +
    facet_grid(course ~ prediction_label, scales = "free_y") +
    geom_histogram(binwidth = .01) +
    guides(fill = "none") +
    labs(x = "Predicted accuracy") +
    scale_x_continuous(breaks = seq(0, 1, by = .25), labels = scales::percent_format()) +
    scale_fill_manual(values = condition_colours)
  
p_acc_pred_dist

ggsave(plot = p_acc_pred_dist, file.path("..", "output", "acc_predicted_distribution.png"),
         device = "png", width = 6, height = 7.5)

rm(p_acc_pred_dist)

Predicted vs observed values

plot_dodge <- function(y, dodge = .1) {
  return (y * (1 + dodge) - dodge/2)
}
p_acc_pred_v_obs <- ggplot(pred_beh, aes(x = predicted_accuracy, y = correct, group = prediction_label, colour = prediction_label, fill = prediction_label)) +
    facet_grid(course ~ prediction_label) +
    geom_point(aes(y = correct),
               position = position_jitter(width = 0, height = .025, seed = 123),
               size = .001, pch = ".", alpha = .1) +
    labs(x = "Predicted accuracy",
         y = "Response accuracy",
         colour = "Prediction method",
         fill = "Prediction method") +
  guides(colour = "none",
         fill = "none") +
    scale_x_continuous(breaks = seq(0, 1, by = .25), labels = scales::percent_format()) +
    scale_y_continuous(breaks = seq(0, 1, by = .25), labels = scales::percent_format()) +
    scale_colour_manual(values = condition_colours) +
    scale_fill_manual(values = condition_colours) +
    coord_cartesian(xlim = c(0, 1), ylim = c(0, 1), clip = "off")

p_acc_pred_v_obs

ggsave(plot = p_acc_pred_v_obs, file.path("..", "output", "acc_predicted_vs_observed.png"),
       device = "png", width = 10, height = 4.5)

rm(p_acc_pred_v_obs)

Prediction error

pred_acc_error <- (pred_beh
                   [, acc_pred_error := predicted_accuracy - correct]
                   [, abs_acc_pred_error := abs(acc_pred_error)])


pred_acc_error_avg <- pred_acc_error[, .(mae = mean(abs_acc_pred_error),
                                         ae_se = sd(abs_acc_pred_error)/.N), 
                                     by = .(course, prediction_label)]

n_obs <- pred_acc_error[, .N, by = .(course, prediction_label)]

Distribution of prediction error:

p_acc_pred_error <- ggplot(pred_acc_error, aes(x = acc_pred_error, fill = prediction_label)) +
  facet_grid(prediction_label ~ course , scales = "free_y") +
  geom_histogram(aes(y = ..density..), binwidth = .01) +
  guides(fill = "none") +
  labs(x = "Accuracy prediction error (predicted - observed)",
       y = "Density") +
  coord_cartesian(xlim = c(-1, 1)) +
  scale_fill_manual(values = condition_colours)

p_acc_pred_error

ggsave(plot = p_acc_pred_error, file.path("..", "output", "acc_prediction_error.png"),
       device = "png", width = 5, height = 7.5)

rm(p_acc_pred_error)

Absolute prediction error

p_abs_acc_pred_error <- ggplot(pred_acc_error, aes(x = abs_acc_pred_error, fill = prediction_label)) +
  facet_grid(prediction_label ~ course , scales = "free_y") +
  geom_histogram(aes(y = ..density..), binwidth = .01) +
  guides(fill = "none") +
  labs(x = "Accuracy prediction error (predicted - observed)",
       y = "Density") +
  coord_cartesian(xlim = c(0, 1)) +
  scale_fill_manual(values = condition_colours)

p_abs_acc_pred_error

ggsave(plot = p_abs_acc_pred_error, file.path("..", "output", "acc_absolute_prediction_error.png"),
       device = "png", width = 5, height = 7.5)

rm(p_abs_acc_pred_error)
ggplot(pred_acc_error_avg, aes(x = prediction_label, y = mae, colour = course)) +
  geom_boxplot(data = pred_acc_error,
               aes(y = abs_acc_pred_error, group = interaction(course, prediction_label)),
               colour = "grey70",
               width = .25,
               outlier.shape = NA,
               position = position_dodge(width = .5)) +
  geom_errorbar(aes(ymin = mae - ae_se, ymax = mae + ae_se), width = 0, position = position_dodge(width = .5)) +
  geom_point(position = position_dodge(width = .5)) +
  coord_cartesian(ylim = c(0, 1)) +
  labs(x = "Method",
       y = "Absolute accuracy prediction error",
       colour = "Course")

Fit a regression model.

French
m_acc_pred_error_gl_file <- file.path("..", "data", "model_fits", "m_acc_pred_error_Grandes_Lignes.rda")

if (file.exists(m_acc_pred_error_gl_file)) {
  load(m_acc_pred_error_gl_file)
} else {
  
  pred_gl_reg <- (
    pred_acc_error
    [course == "French"]
    [sample(.N, 1e6)]
    [, .(prediction_label, abs_acc_pred_error, user_id, fact_id)]
  )
  
  m_acc_pred_error_gl <- lmer(abs_acc_pred_error ~ prediction_label + 
                               (1 | user_id) + (1 | fact_id),
                             data = pred_gl_reg,
                             control = lmerControl(optimizer ="bobyqa"))
    
  save(m_acc_pred_error_gl, file = m_acc_pred_error_gl_file)
}

summary(m_acc_pred_error_gl)
Linear mixed model fit by REML. t-tests use Satterthwaite's method [
lmerModLmerTest]
Formula: abs_acc_pred_error ~ prediction_label + (1 | user_id) + (1 |  
    fact_id)
   Data: pred_gl_reg
Control: lmerControl(optimizer = "bobyqa")

REML criterion at convergence: -2363469

Scaled residuals: 
    Min      1Q  Median      3Q     Max 
-7.2812 -0.4159 -0.0378  0.4040  8.0239 

Random effects:
 Groups   Name        Variance  Std.Dev.
 user_id  (Intercept) 0.0004139 0.02034 
 fact_id  (Intercept) 0.0004236 0.02058 
 Residual             0.0051894 0.07204 
Number of obs: 1000000, groups:  user_id, 40974; fact_id, 22910

Fixed effects:
                                 Estimate Std. Error         df t value
(Intercept)                     4.900e-01  2.656e-04  6.217e+04 1844.99
prediction_labelDomain         -1.095e-02  2.277e-04  9.749e+05  -48.09
prediction_labelFact           -1.586e-02  2.292e-04  9.751e+05  -69.17
prediction_labelLearner        -1.256e-02  2.310e-04  9.757e+05  -54.37
prediction_labelFact & Learner -2.083e-02  2.321e-04  9.760e+05  -89.73
                               Pr(>|t|)    
(Intercept)                      <2e-16 ***
prediction_labelDomain           <2e-16 ***
prediction_labelFact             <2e-16 ***
prediction_labelLearner          <2e-16 ***
prediction_labelFact & Learner   <2e-16 ***
---
Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1

Correlation of Fixed Effects:
            (Intr) prdc_D prdc_F prdc_L
prdctn_lblD -0.429                     
prdctn_lblF -0.420  0.497              
prdctn_lblL -0.416  0.493  0.490       
prdctn_lF&L -0.408  0.490  0.488  0.486

Compare different prediction types to each other:

ht_acc_gl <- glht(m_acc_pred_error_gl, linfct = mcp(prediction_label = "Tukey"))
summary(ht_acc_gl)

     Simultaneous Tests for General Linear Hypotheses

Multiple Comparisons of Means: Tukey Contrasts


Fit: lmer(formula = abs_acc_pred_error ~ prediction_label + (1 | user_id) + 
    (1 | fact_id), data = pred_gl_reg, control = lmerControl(optimizer = "bobyqa"))

Linear Hypotheses:
                                Estimate Std. Error z value Pr(>|z|)    
Domain - Default == 0         -0.0109525  0.0002277 -48.092   <1e-10 ***
Fact - Default == 0           -0.0158576  0.0002292 -69.173   <1e-10 ***
Learner - Default == 0        -0.0125564  0.0002310 -54.367   <1e-10 ***
Fact & Learner - Default == 0 -0.0208270  0.0002321 -89.730   <1e-10 ***
Fact - Domain == 0            -0.0049051  0.0002293 -21.396   <1e-10 ***
Learner - Domain == 0         -0.0016039  0.0002310  -6.944   <1e-10 ***
Fact & Learner - Domain == 0  -0.0098745  0.0002322 -42.532   <1e-10 ***
Learner - Fact == 0            0.0033012  0.0002324  14.202   <1e-10 ***
Fact & Learner - Fact == 0    -0.0049694  0.0002334 -21.290   <1e-10 ***
Fact & Learner - Learner == 0 -0.0082706  0.0002348 -35.222   <1e-10 ***
---
Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
(Adjusted p values reported -- single-step method)

Inspect the model's residuals:

qqnorm(resid(m_acc_pred_error_gl))
qqline(resid(m_acc_pred_error_gl), col = "red")

plot(m_acc_pred_error_gl)

English
m_acc_pred_error_ss_file <- file.path("..", "data", "model_fits", "m_acc_pred_error_Stepping_Stones.rda")

if (file.exists(m_acc_pred_error_ss_file)) {
  load(m_acc_pred_error_ss_file)
} else {
  
  pred_ss_reg <- (
    pred_acc_error
    [course == "English"]
    [sample(.N, 1e6)]
    [, .(prediction_label, abs_acc_pred_error, user_id, fact_id)]
  )
  
  m_acc_pred_error_ss <- lmer(abs_acc_pred_error ~ prediction_label + 
                               (1 | user_id) + (1 | fact_id),
                             data = pred_ss_reg,
                             control = lmerControl(optimizer ="bobyqa")
  )
  
  save(m_acc_pred_error_ss, file = m_acc_pred_error_ss_file)
}

summary(m_acc_pred_error_ss)
Linear mixed model fit by REML. t-tests use Satterthwaite's method [
lmerModLmerTest]
Formula: abs_acc_pred_error ~ prediction_label + (1 | user_id) + (1 |  
    fact_id)
   Data: pred_ss_reg
Control: lmerControl(optimizer = "bobyqa")

REML criterion at convergence: -2365380

Scaled residuals: 
    Min      1Q  Median      3Q     Max 
-7.2429 -0.3226 -0.0256  0.3277  9.3766 

Random effects:
 Groups   Name        Variance  Std.Dev.
 user_id  (Intercept) 0.0004519 0.02126 
 fact_id  (Intercept) 0.0005107 0.02260 
 Residual             0.0050246 0.07088 
Number of obs: 1000000, groups:  user_id, 85899; fact_id, 45529

Fixed effects:
                                 Estimate Std. Error         df t value
(Intercept)                     4.757e-01  2.298e-04  1.102e+05 2069.76
prediction_labelDomain         -1.302e-02  2.281e-04  9.646e+05  -57.10
prediction_labelFact           -1.546e-02  2.291e-04  9.640e+05  -67.46
prediction_labelLearner        -1.432e-02  2.301e-04  9.635e+05  -62.21
prediction_labelFact & Learner -1.969e-02  2.312e-04  9.630e+05  -85.17
                               Pr(>|t|)    
(Intercept)                      <2e-16 ***
prediction_labelDomain           <2e-16 ***
prediction_labelFact             <2e-16 ***
prediction_labelLearner          <2e-16 ***
prediction_labelFact & Learner   <2e-16 ***
---
Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1

Correlation of Fixed Effects:
            (Intr) prdc_D prdc_F prdc_L
prdctn_lblD -0.497                     
prdctn_lblF -0.491  0.499              
prdctn_lblL -0.489  0.497  0.494       
prdctn_lF&L -0.483  0.494  0.492  0.490

Compare different prediction types to each other:

ht_acc_ss <- glht(m_acc_pred_error_ss, linfct = mcp(prediction_label = "Tukey"))
summary(ht_acc_ss)

     Simultaneous Tests for General Linear Hypotheses

Multiple Comparisons of Means: Tukey Contrasts


Fit: lmer(formula = abs_acc_pred_error ~ prediction_label + (1 | user_id) + 
    (1 | fact_id), data = pred_ss_reg, control = lmerControl(optimizer = "bobyqa"))

Linear Hypotheses:
                                Estimate Std. Error z value Pr(>|z|)    
Domain - Default == 0         -0.0130225  0.0002281 -57.099   <1e-04 ***
Fact - Default == 0           -0.0154554  0.0002291 -67.456   <1e-04 ***
Learner - Default == 0        -0.0143165  0.0002301 -62.211   <1e-04 ***
Fact & Learner - Default == 0 -0.0196905  0.0002312 -85.175   <1e-04 ***
Fact - Domain == 0            -0.0024329  0.0002289 -10.630   <1e-04 ***
Learner - Domain == 0         -0.0012941  0.0002299  -5.630   <1e-04 ***
Fact & Learner - Domain == 0  -0.0066681  0.0002310 -28.868   <1e-04 ***
Learner - Fact == 0            0.0011388  0.0002310   4.931   <1e-04 ***
Fact & Learner - Fact == 0    -0.0042352  0.0002319 -18.264   <1e-04 ***
Fact & Learner - Learner == 0 -0.0053740  0.0002329 -23.076   <1e-04 ***
---
Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
(Adjusted p values reported -- single-step method)

Inspect the model's residuals:

qqnorm(resid(m_acc_pred_error_ss))
qqline(resid(m_acc_pred_error_ss), col = "red")

plot(m_acc_pred_error_ss)

Comparison
ht_acc_gl_tidy <- broom::tidy(confint(ht_acc_gl))
ht_acc_ss_tidy <- broom::tidy(confint(ht_acc_ss))
setDT(ht_acc_gl_tidy)
setDT(ht_acc_ss_tidy)

ht_acc_both_tidy <- rbind(ht_acc_gl_tidy[, course := "French"],
                      ht_acc_ss_tidy[, course := "English"])
p_acc_pred_error_comp <- ggplot(ht_acc_both_tidy, aes(x = lhs, y = estimate, ymin = conf.low, ymax = conf.high, colour = course)) +
  geom_hline(yintercept = 0, linetype = "11", colour = "grey60") +
  geom_errorbar(width = 0.1) + 
  geom_point() +
  labs(x = "Linear hypotheses",
       y = "Estimate",
       caption = "Tukey's range test. Error bars show 95% family-wise confidence level.",
       colour = "Course") +
  coord_flip()

p_acc_pred_error_comp

ggsave(plot = p_acc_pred_error_comp, file.path("..", "output", "acc_prediction_error_comparisons.png"),
       device = "png", width = 7.5, height = 5)

rm(p_acc_pred_error_comp)
Summary plot
pred_acc_error_avg[, prediction_rank := frank(-mae), by = .(course)]
annotation_df_ss <- data.table(
  course = rep("English", 10),
  start = c(1, 1, 1, 1,
            2, 2, 2,
            3, 3,
            4
  ),
  end = c(2, 3, 4, 5,
          3, 4, 5,
          4, 5,
          5
  ),
  y = seq(max(pred_acc_error_avg$mae)*1.01 + .01125, max(pred_acc_error_avg$mae)*1.01, by = -.00125),
  label = c("p < .001", "p < .001", "p < .001", "p < .001",
            "p < .001", "p < .001", "p < .001",
            "p < .001", "p < .001",
            "p < .001")
)

annotation_df_gl <- data.table(
  course = rep("French", 10),
  start = c(1, 1, 1, 1,
            2, 2, 2,
            3, 3,
            4
  ),
  end = c(2, 3, 4, 5,
          3, 4, 5,
          4, 5,
          5
  ),
  y = seq(max(pred_acc_error_avg$mae)*1.01 + .01125, max(pred_acc_error_avg$mae)*1.01, by = -.00125),
  label = c("p < .001", "p < .001", "p < .001", "p < .001",
            "p < .001", "p < .001", "p < .001",
            "p < .001", "p < .001",
            "p < .001")
)

annotation_df_acc <- rbind(annotation_df_ss, annotation_df_gl)
annotation_df_acc[, label := factor(label, levels = c("p < .001", "p < .01", "p < .05", "n.s."))]
p_acc_pred_error_summary <- ggplot(pred_acc_error_avg, aes(x = prediction_rank, y = mae)) +
  facet_grid(~ course) +
  geom_line(data = annotation_df_acc,
            aes(x = 1, y = .45, lty = label, alpha = label, colour = NULL)) + # Dummy line to get legend
  geom_line(aes(colour = course, group = course)) +
  geom_errorbar(aes(ymin = mae - ae_se, ymax = mae + ae_se), width = 0) +
  geom_point(aes(colour = course, group = course)) +
  geom_label(aes(label = prediction_label), 
             colour = "black", 
             alpha = .9,
             label.size = NA, 
             nudge_y = -.004) +
  labs(x = NULL,
       y = "Absolute prediction error:\nresponse accuracy",
       colour = "Course") +
  scale_x_continuous(expand = expansion(add = .75), breaks = NULL) +
  scale_colour_manual(values = dataset_colours) +
  scale_linetype_manual(values = c("p < .001" = 1,
                                   "p < .01" = 5,
                                   "p < .05" = 2,
                                   "n.s." = 3),
                        name = "Pairwise comparison:") +
  scale_alpha_manual(values = c("p < .001" = 1,
                                "p < .01" = .75,
                                "p < .05" = .5, 
                                "n.s." = .25), 
                     name = "Pairwise comparison:") +
  guides(colour = "none") +
  ggsignif::geom_signif(data = annotation_df_acc,
                        aes(xmin = start, xmax = end, annotations = "", 
                            y_position = y, lty = label, alpha = label),
                        tip_length = 0,
                        manual = TRUE)  +
  theme(legend.position = "bottom",
        legend.justification = "right")
Warning: Ignoring unknown aesthetics: xmin, xmax, annotations, y_position
p_acc_pred_error_summary

ggsave(file.path("..", "output", "acc_absolute_prediction_error_summary.png"),
       device = "png", width = 10, height = 4)
Improvement

How big was the improvement from worst to best prediction method?

French:

# Absolute change
ht_acc_gl_tidy[lhs == "Fact & Learner - Default", estimate[1]]
[1] -0.02082699
# % change
scales::percent(
  ht_acc_gl_tidy[lhs == "Fact & Learner - Default", estimate[1]] / fixef(m_acc_pred_error_gl)[[1]],
  accuracy = .1)
[1] "-4.3%"
-4.3%

English:

# Absolute change
ht_acc_ss_tidy[lhs == "Fact & Learner - Default", estimate[1]]
[1] -0.01969055
# % change
scales::percent(
  ht_acc_ss_tidy[lhs == "Fact & Learner - Default", estimate[1]] / fixef(m_acc_pred_error_ss)[[1]],
  accuracy = .1)
[1] "-4.1%"
-4.1%

Combined plot

(p_acc_pred_error_summary + p_rt_pred_error_summary) + 
  plot_layout(ncol = 1, guides = "collect") +
  plot_annotation(tag_levels = "A") &
  theme(legend.position = "bottom")
Warning in prettyNum(.Internal(format(x, trim, digits, nsmall, width, 3L, :
'big.mark' and 'decimal.mark' are both '.', which could be confusing

Warning in prettyNum(.Internal(format(x, trim, digits, nsmall, width, 3L, :
'big.mark' and 'decimal.mark' are both '.', which could be confusing

ggsave(file.path("..", "output", "beh_absolute_prediction_error_summary.png"),
       device = "png", width = 10, height = 8)
Warning in prettyNum(.Internal(format(x, trim, digits, nsmall, width, 3L, :
'big.mark' and 'decimal.mark' are both '.', which could be confusing

Warning in prettyNum(.Internal(format(x, trim, digits, nsmall, width, 3L, :
'big.mark' and 'decimal.mark' are both '.', which could be confusing

Session info

sessionInfo()
R version 3.6.3 (2020-02-29)
Platform: x86_64-pc-linux-gnu (64-bit)
Running under: Ubuntu 18.04.6 LTS

Matrix products: default
BLAS:   /usr/lib/x86_64-linux-gnu/blas/libblas.so.3.7.1
LAPACK: /usr/lib/x86_64-linux-gnu/lapack/liblapack.so.3.7.1

locale:
 [1] LC_CTYPE=en_US.UTF-8       LC_NUMERIC=C              
 [3] LC_TIME=en_GB.UTF-8        LC_COLLATE=en_US.UTF-8    
 [5] LC_MONETARY=en_GB.UTF-8    LC_MESSAGES=en_US.UTF-8   
 [7] LC_PAPER=en_GB.UTF-8       LC_NAME=C                 
 [9] LC_ADDRESS=C               LC_TELEPHONE=C            
[11] LC_MEASUREMENT=en_GB.UTF-8 LC_IDENTIFICATION=C       

attached base packages:
[1] stats     graphics  grDevices utils     datasets  methods   base     

other attached packages:
 [1] dplyr_1.0.7       multcomp_1.4-10   TH.data_1.0-10   
 [4] MASS_7.3-51.4     survival_2.44-1.1 mvtnorm_1.1-1    
 [7] lmerTest_3.1-0    lme4_1.1-21       Matrix_1.2-18    
[10] wesanderson_0.3.6 patchwork_1.1.1   ggplot2_3.3.5    
[13] stringr_1.4.0     furrr_0.1.0       future_1.13.0    
[16] purrr_0.3.2       tidyr_1.0.0       data.table_1.13.6
[19] fst_0.9.0        

loaded via a namespace (and not attached):
 [1] zoo_1.8-6           tidyselect_1.1.1    xfun_0.21          
 [4] listenv_0.7.0       splines_3.6.3       lattice_0.20-41    
 [7] colorspace_1.4-1    vctrs_0.3.8         generics_0.1.0     
[10] htmltools_0.3.6     mgcv_1.8-28         yaml_2.2.0         
[13] utf8_1.1.4          rlang_0.4.10        pillar_1.6.3       
[16] nloptr_1.2.1        glue_1.4.2          withr_2.3.0        
[19] DBI_1.1.0           lifecycle_1.0.1     ggsignif_0.5.0     
[22] munsell_0.5.0       gtable_0.3.0        codetools_0.2-16   
[25] evaluate_0.14       labeling_0.3        knitr_1.23         
[28] parallel_3.6.3      fansi_0.4.0         broom_0.5.2        
[31] Rcpp_1.0.6          backports_1.1.4     scales_1.1.1       
[34] jsonlite_1.6        farver_2.1.0        digest_0.6.19      
[37] stringi_1.4.3       numDeriv_2016.8-1.1 grid_3.6.3         
[40] tools_3.6.3         sandwich_2.5-1      magrittr_2.0.1     
[43] tibble_2.1.3        crayon_1.4.1        pkgconfig_2.0.2    
[46] ellipsis_0.3.2      minqa_1.2.4         rmarkdown_2.6      
[49] R6_2.4.0            globals_0.12.4      boot_1.3-25        
[52] nlme_3.1-149        compiler_3.6.3     
LS0tCnRpdGxlOiAiRXZhbHVhdGUgYmVoYXZpb3VyYWwgcHJlZGljdGlvbnMiCmF1dGhvcjogIk1hYXJ0ZW4gdmFuIGRlciBWZWxkZSIKZGF0ZTogIkxhc3QgdXBkYXRlZDogYHIgU3lzLkRhdGUoKWAiCm91dHB1dDoKICBodG1sX25vdGVib29rOgogICAgc21hcnQ6IG5vCiAgICB0b2M6IHllcwogICAgdG9jX2Zsb2F0OiB5ZXMKICBnaXRodWJfZG9jdW1lbnQ6CiAgICB0b2M6IHllcwplZGl0b3Jfb3B0aW9uczogCiAgY2h1bmtfb3V0cHV0X3R5cGU6IGlubGluZQotLS0KCgojIE92ZXJ2aWV3CgpUaGlzIG5vdGVib29rIGV2YWx1YXRlcyB0aGUgYmVoYXZpb3VyYWwgcHJlZGljdGlvbnMgKFJULCBhY2N1cmFjeSkgdGhhdCBmb2xsb3cgZnJvbSB0aGUgcHJlZGljdGVkIHJhdGVzIG9mIGZvcmdldHRpbmcuCldlIHNpbXVsYXRlIHRoZSBiZWhhdmlvdXJhbCBwcmVkaWN0aW9ucyB0aGF0IHRoZSBhZGFwdGl2ZSBmYWN0IGxlYXJuaW5nIHN5c3RlbSB3b3VsZCBoYXZlIG1hZGUsIGlmIGl0IGhhZCB1c2VkIHRoZSBwcmVkaWN0ZWQgcmF0ZSBvZiBmb3JnZXR0aW5nIGFzIHRoZSBzdGFydGluZyBlc3RpbWF0ZSBpbiB0aGUgbGVhcm5pbmcgc2VxdWVuY2UuCgoKIyBTZXR1cAoKYGBge3J9CmxpYnJhcnkoZnN0KQpsaWJyYXJ5KGRhdGEudGFibGUpCmxpYnJhcnkodGlkeXIpCmxpYnJhcnkocHVycnIpCmxpYnJhcnkoZnVycnIpCmxpYnJhcnkoc3RyaW5ncikKbGlicmFyeShnZ3Bsb3QyKQpsaWJyYXJ5KHBhdGNod29yaykKbGlicmFyeSh3ZXNhbmRlcnNvbikKbGlicmFyeShsbWU0KQpsaWJyYXJ5KGxtZXJUZXN0KQpsaWJyYXJ5KG11bHRjb21wKQpgYGAKCmBgYHtyfQpzb3VyY2UoZmlsZS5wYXRoKCIuLiIsICJzY3JpcHRzIiwgIjk5X3NsaW1zdGFtcGVuX21vZGVsX2Z1bnMuUiIpKQpgYGAKCmBgYHtyfQpmdXR1cmU6OnBsYW4oIm11bHRpcHJvY2VzcyIsIHdvcmtlcnMgPSA2KSAjIFNldCB0byBkZXNpcmVkIG51bWJlciBvZiBjb3JlcwpgYGAKCmBgYHtyfQp0aGVtZV9zZXQodGhlbWVfbGlnaHQoYmFzZV9zaXplID0gMTQpICsKICAgICAgICAgICAgdGhlbWUoc3RyaXAudGV4dCA9IGVsZW1lbnRfdGV4dChjb2xvdXIgPSAiYmxhY2siKSkpCgpjb25kaXRpb25fY29sb3VycyA8LSB3ZXNfcGFsZXR0ZSgiRGFyamVlbGluZzEiLCBuID0gNSkKY29uZGl0aW9uX2NvbG91cnNbYygyLCA0LCA1KV0gPC0gY29uZGl0aW9uX2NvbG91cnNbYyg0LCA1LCAyKV0KCmRhdGFzZXRfY29sb3VycyA8LSB3ZXNfcGFsZXR0ZSgiRGFyamVlbGluZzIiLCBuID0gNSlbYygyLCAzKV0KYGBgCgoKIyMgSGVscGVyIGZ1bmN0aW9ucwoKYGBge3J9CmxvYWRfZGF0YV93aXRoX3ByZWRpY3Rpb25zIDwtIGZ1bmN0aW9uIChjb3Vyc2UpIHsKICAKICAjIERhdGEKICBkX2Z1bGwgPC0gcmVhZF9mc3QoZmlsZS5wYXRoKCIuLiIsICJkYXRhIiwgcGFzdGUwKCJmb3JtYXR0ZWRfIiwgc3RyX3JlcGxhY2VfYWxsKGNvdXJzZSwgIiAiLCAiXyIpLCAiLmZzdCIpKSkKICBzZXREVChkX2Z1bGwpCiAgZCA8LSBkX2Z1bGxbIWlzLm5hKGZhY3RfaWQpLCAuKHVzZXJfaWQsIGZhY3RfaWQsIHN0YXJ0X3RpbWUsIHJ0LCBjb3JyZWN0KV0KICBybShkX2Z1bGwpCiAgZ2MoKQogIHNldG9yZGVyKGQsIHVzZXJfaWQsIGZhY3RfaWQsIHN0YXJ0X3RpbWUpCiAgCiAgIyBST0YgcHJlZGljdGlvbnMKICBwcmVkX3VzZXIgPC0gcmVhZF9mc3QoZmlsZS5wYXRoKCIuLiIsICJkYXRhIiwgInByZWRpY3Rpb25zIiwgcGFzdGUwKCJwcmVkX3Zfb2JzX3VzZXJfIiwgc3RyX3JlcGxhY2VfYWxsKGNvdXJzZSwgIiAiLCAiXyIpLCAiLmZzdCIpKSkKICBwcmVkX2ZhY3QgPC0gcmVhZF9mc3QoZmlsZS5wYXRoKCIuLiIsICJkYXRhIiwgInByZWRpY3Rpb25zIiwgcGFzdGUwKCJwcmVkX3Zfb2JzX2ZhY3RfIiwgc3RyX3JlcGxhY2VfYWxsKGNvdXJzZSwgIiAiLCAiXyIpLCAiLmZzdCIpKSkKICBwcmVkX2ZhY3RfdXNlciA8LSByZWFkX2ZzdChmaWxlLnBhdGgoIi4uIiwgImRhdGEiLCAicHJlZGljdGlvbnMiLCBwYXN0ZTAoInByZWRfZmFjdF9hbmRfdXNlcl8iLCBzdHJfcmVwbGFjZV9hbGwoY291cnNlLCAiICIsICJfIiksICIuZnN0IikpKQogIHNldERUKHByZWRfdXNlcikKICBzZXREVChwcmVkX2ZhY3QpCiAgc2V0RFQocHJlZF9mYWN0X3VzZXIpCiAgCiAgIyBSZW1vdmUgTkEgYW5kIGR1cGxpY2F0ZXMKICBwcmVkX3VzZXIgPC0gdW5pcXVlKHByZWRfdXNlclshaXMubmEoYWxwaGEpXSkKICBwcmVkX2ZhY3QgPC0gdW5pcXVlKHByZWRfZmFjdFshaXMubmEoYWxwaGEpXSkKICBwcmVkX2ZhY3RfdXNlciA8LSB1bmlxdWUocHJlZF9mYWN0X3VzZXJbIWlzLm5hKGFscGhhKV0pCiAgCiAgIyBNYWtlIERvbWFpbiBwcmVkaWN0aW9uCiAgcHJlZF9kb21haW4gPC0gbWVhbih1bmlxdWUocHJlZF9mYWN0LCBieSA9IGMoImZhY3RfaWQiKSkkcHJlZF9mYWN0KQogIHByZWRfZGVmYXVsdCA8LSAwLjMKICAKICAjIENvbWJpbmUKICBzZXRuYW1lcyhwcmVkX3VzZXIsICJuX3RyYWluX29icyIsICJuX3RyYWluX29ic191c2VyIikKICBzZXRuYW1lcyhwcmVkX2ZhY3QsICJuX3RyYWluX29icyIsICJuX3RyYWluX29ic19mYWN0IikKICBwcmVkX2FsbCA8LSBtZXJnZShwcmVkX3VzZXIsIHByZWRfZmFjdCwgYnkgPSBjKCJ1c2VyX2lkIiwgImZhY3RfaWQiLCAiYWxwaGEiLCAibl9yZXBzIiksIGFsbCA9IFRSVUUpCiAgcHJlZF9hbGwgPC0gbWVyZ2UocHJlZF9hbGwsIHByZWRfZmFjdF91c2VyLCBieSA9IGMoInVzZXJfaWQiLCAiZmFjdF9pZCIsICJhbHBoYSIpLCBhbGwgPSBUUlVFKQogIHByZWRfYWxsWywgcHJlZF9kZWZhdWx0IDo9IHByZWRfZGVmYXVsdF0KICBwcmVkX2FsbFssIHByZWRfZG9tYWluIDo9IHByZWRfZG9tYWluXQoKICBkX3ByZXAgPC0gZFssIC4odXNlcl9pZCwKICAgICAgICAgICAgICAgICAgZmFjdF9pZCwKICAgICAgICAgICAgICAgICAgdGV4dCA9ICIiLAogICAgICAgICAgICAgICAgICBzdGFydF90aW1lLAogICAgICAgICAgICAgICAgICBydCwKICAgICAgICAgICAgICAgICAgY29ycmVjdCwKICAgICAgICAgICAgICAgICAgdGhyZXNob2xkID0gLTAuOCldCiAgCiAgZF9wcmVkIDwtIG1lcmdlKHByZWRfYWxsLCBkX3ByZXAsIGJ5ID0gYygidXNlcl9pZCIsICJmYWN0X2lkIikpCiAgCiAgcmV0dXJuKGRfcHJlZCkKfQoKcHJlZGljdF9iZWhhdmlvdXIgPC0gZnVuY3Rpb24gKGQpIHsKICAKICAgICMgUHJvY2VzcyB0aGUgZGF0YSBpbiBtYW5hZ2VhYmxlIGNodW5rcwogICAgY2h1bmtfc2l6ZSA8LSAxZTQKICAgIG9icyA8LSB1bmlxdWUoZFssIC4odXNlcl9pZCwgZmFjdF9pZCldKQogICAgb2JzX2NodW5rcyA8LSBjKHNlcSgxLCBucm93KG9icyksIGJ5ID0gY2h1bmtfc2l6ZSksIG5yb3cob2JzKSsxKQogICAgCiAgICBwcmVkX2JlaCA8LSBtYXBfZGZyKDE6KGxlbmd0aChvYnNfY2h1bmtzKS0xKSwgZnVuY3Rpb24gKGkpIHsKICAgICAgCiAgICAgIG1zZyA8LSBwYXN0ZSgiQ2h1bmsiLCBpLCAiLyIsIGxlbmd0aChvYnNfY2h1bmtzKS0xKQogICAgICBzeXN0ZW0ocGFzdGUoImVjaG8iLCBtc2cpKQogICAgICAKICAgICAgb2JzX2kgPC0gb2JzW29ic19jaHVua3NbaV06b2JzX2NodW5rc1tpKzFdLTFdCiAgICAgIGRfaSA8LSBkW29ic19pLCBvbiA9IC4odXNlcl9pZCwgZmFjdF9pZCldCiAgICAgIGRfaV9saXN0IDwtIHNwbGl0KGRfaSwgYnkgPSBjKCJ1c2VyX2lkIiwgImZhY3RfaWQiKSwgZHJvcCA9IFRSVUUpCiAgICAgIAogICAgICBwcmVkX2JlaF9pIDwtIGZ1dHVyZV9tYXBfZGZyKGRfaV9saXN0LCBmdW5jdGlvbiAobGVhcm5fc2VxKSB7CiAgICAgICAgCiAgICAgICAgIyBPcmdhbmlzZSBwcmVkaWN0aW9ucyBmb3IgdGhpcyBzZXF1ZW5jZQogICAgICAgIGxlYXJuX3NlcV9wcmVkcyA8LSBwaXZvdF9sb25nZXIoCiAgICAgICAgICBsZWFybl9zZXFbMSxdLCAKICAgICAgICAgIHByZWRfdXNlcjpwcmVkX2RvbWFpbiwKICAgICAgICAgIG5hbWVzX3RvID0gInByZWRpY3Rpb25fdHlwZSIsCiAgICAgICAgICBuYW1lc19wcmVmaXggPSAicHJlZF8iLAogICAgICAgICAgdmFsdWVzX3RvID0gInByZWRpY3RlZF9hbHBoYSIKICAgICAgICApCiAgICAgICAgc2V0RFQobGVhcm5fc2VxX3ByZWRzKQogICAgICAgIAogICAgICAgICMgTG9vayBvbmx5IGF0IHRyaWFsIDMgaW4gZWFjaCBzZXF1ZW5jZQogICAgICAgIHRyaWFsIDwtIGxlYXJuX3NlcVszLF0KICAgICAgICBkZWxheSA8LSBsZWFybl9zZXFbMjozLCBkaWZmKHN0YXJ0X3RpbWUpXQogICAgICAgIAogICAgICAgICMgQ2FsY3VsYXRlIGJlaGF2aW91cmFsIHByZWRpY3Rpb25zIGZvciBlYWNoIHByZWRpY3Rpb24gbWV0aG9kCiAgICAgICAgbWFwX2RmcihzZXEobnJvdyhsZWFybl9zZXFfcHJlZHMpKSwgZnVuY3Rpb24gKGopIHsKICAgICAgICAgIAogICAgICAgICAgcHJlZGljdGlvbl90eXBlIDwtIGxlYXJuX3NlcV9wcmVkc1tqLCBwcmVkaWN0aW9uX3R5cGVdCiAgICAgICAgICBwcmVkaWN0ZWRfYWxwaGEgPC0gbGVhcm5fc2VxX3ByZWRzW2osIHByZWRpY3RlZF9hbHBoYV0KICAgIAogICAgICAgICAgcHJlZGljdGVkX2FjdGl2YXRpb24gPC0gTkEKICAgICAgICAgIHByZWRpY3RlZF9hY2N1cmFjeSA8LSBOQQogICAgICAgICAgcHJlZGljdGVkX3J0IDwtIE5BCiAgICAKICAgICAgICAgIGlmICghaXMubmEocHJlZGljdGVkX2FscGhhKSkgewogICAgICAgICAgICAKICAgICAgICAgICAgcHJlZGljdGVkX2FjdGl2YXRpb24gPC0gY2FsY3VsYXRlX2FjdGl2YXRpb24oCiAgICAgICAgICAgICAgdGltZSA9IHRyaWFsWywgc3RhcnRfdGltZV0sCiAgICAgICAgICAgICAgaWQgPSB0cmlhbFssIGZhY3RfaWRdLAogICAgICAgICAgICAgIGZhY3RhbHBoYSA9IHByZWRpY3RlZF9hbHBoYSwKICAgICAgICAgICAgICByZXNwb25zZXMgPSBsZWFybl9zZXFbMToyLF0KICAgICAgICAgICAgKQogICAgICAKICAgICAgICAgICAgcHJlZGljdGVkX2FjY3VyYWN5IDwtIHBfcmVjYWxsKAogICAgICAgICAgICAgIGFjdGl2YXRpb24gPSBwcmVkaWN0ZWRfYWN0aXZhdGlvbiwKICAgICAgICAgICAgICB0aHJlc2hvbGQgPSAtMC44LAogICAgICAgICAgICAgIGFjdGl2YXRpb25fbm9pc2UgPSAwLjUKICAgICAgICAgICAgKQogICAgICAgICAgICAKICAgICAgICAgICAgcHJlZGljdGVkX3J0IDwtIGVzdGltYXRlX3JlYWN0aW9uX3RpbWVfZnJvbV9hY3RpdmF0aW9uKAogICAgICAgICAgICAgIGFjdGl2YXRpb24gPSBwcmVkaWN0ZWRfYWN0aXZhdGlvbiwKICAgICAgICAgICAgICByZWFkaW5nX3RpbWUgPSAzMDAKICAgICAgICAgICAgKQogICAgICAgICAgfQogICAgICAgICAgCiAgICAgICAgICByZXR1cm4oCiAgICAgICAgICAgIGxpc3QoCiAgICAgICAgICAgICAgdXNlcl9pZCA9IHRyaWFsWywgdXNlcl9pZF0sCiAgICAgICAgICAgICAgZmFjdF9pZCA9IHRyaWFsWywgZmFjdF9pZF0sCiAgICAgICAgICAgICAgZGVsYXkgPSBkZWxheSwKICAgICAgICAgICAgICBjb3JyZWN0ID0gdHJpYWxbLCBjb3JyZWN0XSwKICAgICAgICAgICAgICBydCA9IHRyaWFsWywgcnRdLAogICAgICAgICAgICAgIHByZWRpY3Rpb25fdHlwZSA9IHByZWRpY3Rpb25fdHlwZSwKICAgICAgICAgICAgICBwcmVkaWN0ZWRfYWxwaGEgPSBwcmVkaWN0ZWRfYWxwaGEsCiAgICAgICAgICAgICAgcHJlZGljdGVkX2FjdGl2YXRpb24gPSBwcmVkaWN0ZWRfYWN0aXZhdGlvbiwKICAgICAgICAgICAgICBwcmVkaWN0ZWRfYWNjdXJhY3kgPSBwcmVkaWN0ZWRfYWNjdXJhY3ksCiAgICAgICAgICAgICAgcHJlZGljdGVkX3J0ID0gcHJlZGljdGVkX3J0CiAgICAgICAgICAgICkKICAgICAgICAgICkKICAgICAgICAgIAogICAgICAgIH0pCiAgICAgIH0pCiAgICB9KQogICAgCiAgICBzZXREVChwcmVkX2JlaCkKICAgIAogICAgIyBSZW1vdmUgcm93cyB3aXRob3V0IHByZWRpY3Rpb24KICAgIHByZWRfYmVoIDwtIHByZWRfYmVoWyFpcy5uYShwcmVkaWN0ZWRfYWxwaGEpXQogICAgcHJlZF9iZWggPC0gcHJlZF9iZWhbIWlzLmluZmluaXRlKHByZWRpY3RlZF9ydCldCiAgICAKICAgICMgU2V0IHByb3BlciBjb25kaXRpb24gbGFiZWxzCiAgICBjb25kaXRpb25fbGFiZWxzIDwtIGRhdGEudGFibGUoCiAgICAgIHByZWRpY3Rpb25fdHlwZSA9IGMoImRlZmF1bHQiLCAiZG9tYWluIiwgImZhY3QiLCAidXNlciIsICJmYWN0X3VzZXIiKSwKICAgICAgcHJlZGljdGlvbl9sYWJlbCA9IGZhY3RvcigKICAgICAgICBjKCJEZWZhdWx0IiwgIkRvbWFpbiIsICJGYWN0IiwgIkxlYXJuZXIiLCAiRmFjdCAmIExlYXJuZXIiKSwKICAgICAgICBsZXZlbHMgPSBjKCJEZWZhdWx0IiwgIkRvbWFpbiIsICJGYWN0IiwgIkxlYXJuZXIiLCAiRmFjdCAmIExlYXJuZXIiKQogICAgICApCiAgICApCiAgICBwcmVkX2JlaCA8LSBwcmVkX2JlaFtjb25kaXRpb25fbGFiZWxzLCBvbiA9IC4ocHJlZGljdGlvbl90eXBlKV0KICAgIAogICAgCiAgICByZXR1cm4ocHJlZF9iZWgpCn0KYGBgCgoKIyBDYWxjdWxhdGUgcHJlZGljdGlvbnMKCkxvYWQgdGVzdCBzZXQgZGF0YSB3aXRoIHJhdGUgb2YgZm9yZ2V0dGluZyBwcmVkaWN0aW9uczoKYGBge3J9CnByZWRfZ2wgPC0gbG9hZF9kYXRhX3dpdGhfcHJlZGljdGlvbnMoIkdyYW5kZXMgTGlnbmVzIikKcHJlZF9zcyA8LSBsb2FkX2RhdGFfd2l0aF9wcmVkaWN0aW9ucygiU3RlcHBpbmcgU3RvbmVzIikKYGBgCgpDYWxjdWxhdGUgYmVoYXZpb3VyYWwgcHJlZGljdGlvbnMgZm9yIHRyaWFsIDM6CmBgYHtyfQpwcmVkX2dsX2JlaF9wYXRoIDwtIGZpbGUucGF0aCgiLi4iLCAiZGF0YSIsICJwcmVkaWN0aW9ucyIsICJwcmVkX2JlaGF2aW91cl9nbC5mc3QiKQpwcmVkX3NzX2JlaF9wYXRoIDwtIGZpbGUucGF0aCgiLi4iLCAiZGF0YSIsICJwcmVkaWN0aW9ucyIsICJwcmVkX2JlaGF2aW91cl9zcy5mc3QiKQoKaWYgKCFmaWxlLmV4aXN0cyhwcmVkX2dsX2JlaF9wYXRoKSkgewogIHByZWRfZ2xfYmVoIDwtIHByZWRpY3RfYmVoYXZpb3VyKHByZWRfZ2wpCiAgd3JpdGVfZnN0KHByZWRfZ2xfYmVoLCBwcmVkX2dsX2JlaF9wYXRoKQp9IGVsc2UgewogIHByZWRfZ2xfYmVoIDwtIHJlYWRfZnN0KHByZWRfZ2xfYmVoX3BhdGgpCiAgc2V0RFQocHJlZF9nbF9iZWgpCn0KCmlmICghZmlsZS5leGlzdHMocHJlZF9zc19iZWhfcGF0aCkpIHsKICBwcmVkX3NzX2JlaCA8LSBwcmVkaWN0X2JlaGF2aW91cihwcmVkX3NzKQogIHdyaXRlX2ZzdChwcmVkX3NzX2JlaCwgcHJlZF9zc19iZWhfcGF0aCkKfSBlbHNlIHsKICBwcmVkX3NzX2JlaCA8LSByZWFkX2ZzdChwcmVkX3NzX2JlaF9wYXRoKQogIHNldERUKHByZWRfc3NfYmVoKQp9CmBgYAoKCgoKYGBge3J9CnByZWRfYmVoIDwtIHJiaW5kKHByZWRfZ2xfYmVoWywgY291cnNlIDo9ICJGcmVuY2giXSwKICAgICAgICAgICAgICAgICAgcHJlZF9zc19iZWhbLCBjb3Vyc2UgOj0gIkVuZ2xpc2giXSkKCnJtKHByZWRfZ2xfYmVoLCBwcmVkX3NzX2JlaCkKZ2MoKQpgYGAKCgojIEFjdGl2YXRpb24KCmBgYHtyfQpwX2FjdF9kaXN0IDwtIGdncGxvdChwcmVkX2JlaFtiZXR3ZWVuKHByZWRpY3RlZF9hY3RpdmF0aW9uLCAtMiwgMCldLCBhZXMoeCA9IHByZWRpY3RlZF9hY3RpdmF0aW9uLCBmaWxsID0gcHJlZGljdGlvbl9sYWJlbCkpICsKICBmYWNldF9ncmlkKGNvdXJzZSB+IHByZWRpY3Rpb25fbGFiZWwsIHNjYWxlcyA9ICJmcmVlX3kiKSArCiAgZ2VvbV9oaXN0b2dyYW0oYWVzKHkgPSAuLmRlbnNpdHkuLiksIGJpbndpZHRoID0gLjAxKSArCiAgZ3VpZGVzKGZpbGwgPSAibm9uZSIpICsKICBsYWJzKHggPSAiUlQgKGluIHMpIiwKICAgICAgIHkgPSAiRGVuc2l0eSIpICsKICBzY2FsZV9maWxsX21hbnVhbCh2YWx1ZXMgPSBjb25kaXRpb25fY29sb3VycykKCiMgcF9hY3RfZGlzdAoKZ2dzYXZlKHBsb3QgPSBwX2FjdF9kaXN0LCBmaWxlLnBhdGgoIi4uIiwgIm91dHB1dCIsICJhY3RpdmF0aW9uX2Rpc3RyaWJ1dGlvbi5wbmciKSwKICAgICAgIGRldmljZSA9ICJwbmciLCB3aWR0aCA9IDcuNSwgaGVpZ2h0ID0gNC41KQoKcm0ocF9hY3RfZGlzdCkKYGBgCgoKCiMgUmVzcG9uc2UgdGltZQoKRGlzdHJpYnV0aW9uIG9mIG9ic2VydmVkIGNvcnJlY3QgUlQgKHRydW5jYXRlZCBhdCAyNSBzZWNvbmRzIGZvciByZWFkYWJpbGl0eSk6CmBgYHtyfQpwX3J0X2Rpc3QgPC0gZ2dwbG90KHByZWRfYmVoW2NvcnJlY3QgPT0gMSAmIGJldHdlZW4ocnQsIDAsIDI1MDAwKV0sIGFlcyh4ID0gcnQvMTAwMCkpICsKICBmYWNldF9ncmlkKGNvdXJzZSB+IC4sIHNjYWxlcyA9ICJmcmVlX3kiKSArCiAgZ2VvbV9oaXN0b2dyYW0oYWVzKHkgPSAuLmRlbnNpdHkuLiksIGJpbndpZHRoID0gLjEpICsKICBndWlkZXMoZmlsbCA9ICJub25lIikgKwogIGxhYnMoeCA9ICJSVCAoaW4gcykiLAogICAgICAgeSA9ICJEZW5zaXR5IikKCiMgcF9ydF9kaXN0CgpnZ3NhdmUocGxvdCA9IHBfcnRfZGlzdCwgZmlsZS5wYXRoKCIuLiIsICJvdXRwdXQiLCAicnRfb2JzZXJ2ZWRfZGlzdHJpYnV0aW9uLnBuZyIpLAogICAgICAgZGV2aWNlID0gInBuZyIsIHdpZHRoID0gNiwgaGVpZ2h0ID0gNikKCnJtKHBfcnRfZGlzdCkKYGBgCgojIyBQcmVkaWN0ZWQgcmVzcG9uc2UgdGltZQoKIyMjIERpc3RyaWJ1dGlvbiBvZiBwcmVkaWN0aW9ucwoKYGBge3J9CnBfcnRfcHJlZF9kaXN0IDwtIGdncGxvdChwcmVkX2JlaCwgYWVzKHggPSBwcmVkaWN0ZWRfcnQvMTAwMCkpICsKICBmYWNldF9ncmlkKGNvdXJzZSB+IC4sIHNjYWxlcyA9ICJmcmVlX3kiKSArCiAgZ2VvbV9oaXN0b2dyYW0oYWVzKHkgPSAuLmRlbnNpdHkuLiksIGJpbndpZHRoID0gLjEpICsKICBndWlkZXMoZmlsbCA9ICJub25lIikgKwogIGxhYnMoeCA9ICJQcmVkaWN0ZWQgUlQgKGluIHMpIiwKICAgICAgIHkgPSAiRGVuc2l0eSIpCgojIHBfcnRfcHJlZF9kaXN0CgpnZ3NhdmUocGxvdCA9IHBfcnRfcHJlZF9kaXN0LCBmaWxlLnBhdGgoIi4uIiwgIm91dHB1dCIsICJydF9wcmVkaWN0ZWRfZGlzdHJpYnV0aW9uLnBuZyIpLAogICAgICAgZGV2aWNlID0gInBuZyIsIHdpZHRoID0gNiwgaGVpZ2h0ID0gNikKCnJtKHBfcnRfcHJlZF9kaXN0KQpgYGAKCiMjIyBQcmVkaWN0ZWQgdnMgb2JzZXJ2ZWQgdmFsdWVzCgpDYWxjdWxhdGUgYWJzb2x1dGUgcHJlZGljdGlvbiBlcnJvcjoKYGBge3J9CnByZWRfcnRfZXJyb3IgPC0gKHByZWRfYmVoCiAgICAgICAgICAgICAgICAgIFtjb3JyZWN0ID09IDFdCiAgICAgICAgICAgICAgICAgIFtiZXR3ZWVuKHJ0LCAwLCAyNTAwMCldCiAgICAgICAgICAgICAgICAgIFssIHJ0X3ByZWRfZXJyb3IgOj0gcHJlZGljdGVkX3J0IC0gcnRdCiAgICAgICAgICAgICAgICAgIFssIGFic19ydF9wcmVkX2Vycm9yIDo9IGFicyhydF9wcmVkX2Vycm9yKV0KKQoKcHJlZF9ydF9lcnJvcl9hdmcgPC0gcHJlZF9ydF9lcnJvclssIC4obWFlID0gbWVhbihhYnNfcnRfcHJlZF9lcnJvciksCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIGFlX3NlID0gc2QoYWJzX3J0X3ByZWRfZXJyb3IpLy5OKSwgCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgYnkgPSAuKGNvdXJzZSwgcHJlZGljdGlvbl9sYWJlbCldCiAgCm5fb2JzIDwtIHByZWRfcnRfZXJyb3JbLCAuTiwgYnkgPSAuKGNvdXJzZSwgcHJlZGljdGlvbl9sYWJlbCldCmBgYAoKCmBgYHtyfQojIHBsb3RfcmFuZ2UgPC0gcmFuZ2UocHJlZF9iZWgkcHJlZGljdGVkX3J0LzEwMDAsIG5hLnJtID0gVFJVRSkKcGxvdF9yYW5nZSA8LSBjKDAsIDEwKQpwbG90X2JyZWFrcyA8LSBzZXEoMCwgMTAsIGJ5ID0gMikKIyAKIyBwbG90X3JhbmdlIDwtIHF1YW50aWxlKHByZWRfYmVoJHByZWRpY3RlZF9ydC8xMDAwLCBjKC4wMDUsIC45OTUpKQoKcF9ydF9wcmVkX3Zfb2JzIDwtIGdncGxvdChwcmVkX2JlaFtjb3JyZWN0ID09IDFdLCBhZXMoeCA9IHByZWRpY3RlZF9ydC8xMDAwLCB5ID0gcnQvMTAwMCwgY29sb3VyID0gcHJlZGljdGlvbl9sYWJlbCkpICsKICBmYWNldF9ncmlkKGNvdXJzZSB+IHByZWRpY3Rpb25fbGFiZWwpICsKICBnZW9tX2FibGluZShzbG9wZSA9IDEsIGludGVyY2VwdCA9IDAsIGx0eSA9IDMsIGFscGhhID0gMC43NSkgKwogIGdlb21fcG9pbnQoYWxwaGEgPSAuMSwgc2l6ZSA9IC4xLCBwY2ggPSAiLiIpICsKICBnZW9tX3Ntb290aChtZXRob2QgPSAibG0iLCBmb3JtdWxhID0geSB+IHgsIGNvbG91ciA9ICJibGFjayIpICsKICBnZW9tX2xhYmVsKGRhdGEgPSBwcmVkX3J0X2Vycm9yX2F2ZywKICAgICAgICAgICAgYWVzKGxhYmVsID0gcGFzdGUoIk1BRSA9IiwgZm9ybWF0QyhtYWUvMTAwMCwgZGlnaXRzID0gMywgZmxhZyA9ICIjIikpKSwKICAgICAgICAgICAgeCA9IHBsb3RfcmFuZ2VbMl0sIHkgPSBwbG90X3JhbmdlWzFdLAogICAgICAgICAgICBoanVzdCA9IDEsIGNvbG91ciA9ICJOQSIsIHNpemUgPSAzLAogICAgICAgICAgICBhbHBoYSA9IC45LAogICAgICAgICAgICBsYWJlbC5zaXplID0gTkEpICsKICBnZW9tX3RleHQoZGF0YSA9IHByZWRfcnRfZXJyb3JfYXZnLAogICAgICAgICAgICBhZXMobGFiZWwgPSBwYXN0ZSgiTUFFID0iLCBmb3JtYXRDKG1hZS8xMDAwLCBkaWdpdHMgPSAzLCBmbGFnID0gIiMiKSkpLAogICAgICAgICAgICB4ID0gcGxvdF9yYW5nZVsyXSwgeSA9IHBsb3RfcmFuZ2VbMV0sCiAgICAgICAgICAgIGhqdXN0ID0gMSwgY29sb3VyID0gImJsYWNrIiwgc2l6ZSA9IDMpICsKICBnZW9tX2xhYmVsKGRhdGEgPSBuX29icywKICAgICAgICAgICAgYWVzKGxhYmVsID0gcGFzdGUoIm4gPSIsIHNjYWxlczo6Y29tbWEoTikpKSwKICAgICAgICAgICAgeCA9IHBsb3RfcmFuZ2VbMl0sCiAgICAgICAgICAgIHkgPSBwbG90X3JhbmdlWzJdLAogICAgICAgICAgICBoanVzdCA9IDEsIGNvbG91ciA9ICJOQSIsIHNpemUgPSAzLAogICAgICAgICAgICBhbHBoYSA9IC45LAogICAgICAgICAgICBsYWJlbC5zaXplID0gTkEpICsKICBnZW9tX3RleHQoZGF0YSA9IG5fb2JzLAogICAgICAgICAgICBhZXMobGFiZWwgPSBwYXN0ZSgibiA9Iiwgc2NhbGVzOjpjb21tYShOKSkpLAogICAgICAgICAgICB4ID0gcGxvdF9yYW5nZVsyXSwKICAgICAgICAgICAgeSA9IHBsb3RfcmFuZ2VbMl0sCiAgICAgICAgICAgIGhqdXN0ID0gMSwgY29sb3VyID0gImJsYWNrIiwgc2l6ZSA9IDMpICsKICBndWlkZXMoY29sb3VyID0gIm5vbmUiKSArCiAgbGFicyh4ID0gIlByZWRpY3RlZCBSVCAocykiLAogICAgICAgeSA9ICJPYnNlcnZlZCBSVCAocykiKSArCiAgY29vcmRfZml4ZWQocmF0aW8gPSAxLCB4bGltID0gcGxvdF9yYW5nZSwgeWxpbSA9IHBsb3RfcmFuZ2UpICsKICBzY2FsZV94X2NvbnRpbnVvdXMoYnJlYWtzID0gcGxvdF9icmVha3MpICsKICBzY2FsZV95X2NvbnRpbnVvdXMoYnJlYWtzID0gcGxvdF9icmVha3MpICsKICBzY2FsZV9jb2xvdXJfbWFudWFsKHZhbHVlcyA9IGNvbmRpdGlvbl9jb2xvdXJzKQoKcF9ydF9wcmVkX3Zfb2JzCgpnZ3NhdmUocGxvdCA9IHBfcnRfcHJlZF92X29icywgZmlsZS5wYXRoKCIuLiIsICJvdXRwdXQiLCAicnRfcHJlZGljdGVkX3ZzX29ic2VydmVkLnBuZyIpLAogICAgICAgZGV2aWNlID0gInBuZyIsIHdpZHRoID0gMTAsIGhlaWdodCA9IDQuNSkKCnJtKHBfcnRfcHJlZF92X29icykKYGBgCgojIyMjIFByZWRpY3Rpb24gZXJyb3IKCkRpc3RyaWJ1dGlvbiBvZiBwcmVkaWN0aW9uIGVycm9yICh0cnVuY2F0ZWQgdG8gWy01LCA1XSBmb3IgcmVhZGFiaWxpdHkpOgpgYGB7cn0KcF9ydF9wcmVkX2Vycm9yIDwtIGdncGxvdChwcmVkX3J0X2Vycm9yLCBhZXMoeCA9IHJ0X3ByZWRfZXJyb3IvMTAwMCwgZmlsbCA9IHByZWRpY3Rpb25fbGFiZWwpKSArCiAgZmFjZXRfZ3JpZChwcmVkaWN0aW9uX2xhYmVsIH4gY291cnNlICwgc2NhbGVzID0gImZyZWVfeSIpICsKICBnZW9tX2hpc3RvZ3JhbShhZXMoeSA9IC4uZGVuc2l0eS4uKSwgYmlud2lkdGggPSAuMSkgKwogIGd1aWRlcyhmaWxsID0gIm5vbmUiKSArCiAgbGFicyh4ID0gIlJUIHByZWRpY3Rpb24gZXJyb3IgaW4gcyAocHJlZGljdGVkIC0gb2JzZXJ2ZWQpIiwKICAgICAgIHkgPSAiRGVuc2l0eSIpICsKICBjb29yZF9jYXJ0ZXNpYW4oeGxpbSA9IGMoLTUsIDUpKSArCiAgc2NhbGVfZmlsbF9tYW51YWwodmFsdWVzID0gY29uZGl0aW9uX2NvbG91cnMpCgpwX3J0X3ByZWRfZXJyb3IKCmdnc2F2ZShwbG90ID0gcF9ydF9wcmVkX2Vycm9yLCBmaWxlLnBhdGgoIi4uIiwgIm91dHB1dCIsICJydF9wcmVkaWN0aW9uX2Vycm9yLnBuZyIpLAogICAgICAgZGV2aWNlID0gInBuZyIsIHdpZHRoID0gNSwgaGVpZ2h0ID0gNy41KQoKcm0ocF9ydF9wcmVkX2Vycm9yKQpgYGAKCiMjIyMgQWJzb2x1dGUgcHJlZGljdGlvbiBlcnJvcgoKYGBge3J9CnBfcnRfYWJzX3ByZWRfZXJyb3IgPC0gZ2dwbG90KHByZWRfcnRfZXJyb3IsIGFlcyh4ID0gYWJzX3J0X3ByZWRfZXJyb3IvMTAwMCwgZmlsbCA9IHByZWRpY3Rpb25fbGFiZWwpKSArCiAgZmFjZXRfZ3JpZChwcmVkaWN0aW9uX2xhYmVsIH4gY291cnNlLCBzY2FsZXMgPSAiZnJlZV95IikgKwogIGdlb21faGlzdG9ncmFtKGFlcyh5ID0gLi5kZW5zaXR5Li4pLCBiaW53aWR0aCA9IC4xKSArCiAgZ3VpZGVzKGZpbGwgPSAibm9uZSIpICsKICBsYWJzKHggPSAiUlQgcHJlZGljdGlvbiBlcnJvciBpbiBzIChwcmVkaWN0ZWQgLSBvYnNlcnZlZCkiLAogICAgICAgeSA9ICJEZW5zaXR5IikgKwogIGNvb3JkX2NhcnRlc2lhbih4bGltID0gYygwLCA1KSkgKwogIHNjYWxlX2ZpbGxfbWFudWFsKHZhbHVlcyA9IGNvbmRpdGlvbl9jb2xvdXJzKQoKcF9ydF9hYnNfcHJlZF9lcnJvcgoKZ2dzYXZlKHBsb3QgPSBwX3J0X2Fic19wcmVkX2Vycm9yLCBmaWxlLnBhdGgoIi4uIiwgIm91dHB1dCIsICJydF9hYnNvbHV0ZV9wcmVkaWN0aW9uX2Vycm9yLnBuZyIpLAogICAgICAgZGV2aWNlID0gInBuZyIsIHdpZHRoID0gNSwgaGVpZ2h0ID0gNy41KQoKcm0ocF9ydF9hYnNfcHJlZF9lcnJvcikKYGBgCgpgYGB7cn0KZ2dwbG90KHByZWRfcnRfZXJyb3JfYXZnLCBhZXMoeCA9IHByZWRpY3Rpb25fbGFiZWwsIHkgPSBtYWUvMTAwMCwgY29sb3VyID0gY291cnNlKSkgKwogIGdlb21fYm94cGxvdChkYXRhID0gcHJlZF9ydF9lcnJvciwKICAgICAgICAgICAgICAgYWVzKHkgPSBhYnNfcnRfcHJlZF9lcnJvci8xMDAwLCBncm91cCA9IGludGVyYWN0aW9uKGNvdXJzZSwgcHJlZGljdGlvbl9sYWJlbCkpLAogICAgICAgICAgICAgICBjb2xvdXIgPSAiZ3JleTcwIiwKICAgICAgICAgICAgICAgd2lkdGggPSAuMjUsCiAgICAgICAgICAgICAgIG91dGxpZXIuc2hhcGUgPSBOQSwKICAgICAgICAgICAgICAgcG9zaXRpb24gPSBwb3NpdGlvbl9kb2RnZSh3aWR0aCA9IC41KSkgKwogIGdlb21fZXJyb3JiYXIoYWVzKHltaW4gPSBtYWUvMTAwMCAtIGFlX3NlLzEwMDAsIHltYXggPSBtYWUvMTAwMCArIGFlX3NlLzEwMDApLCB3aWR0aCA9IDAsIHBvc2l0aW9uID0gcG9zaXRpb25fZG9kZ2Uod2lkdGggPSAuNSkpICsKICBnZW9tX3BvaW50KHBvc2l0aW9uID0gcG9zaXRpb25fZG9kZ2Uod2lkdGggPSAuNSkpICsKICBjb29yZF9jYXJ0ZXNpYW4oeWxpbSA9IGMoMCwgMi41KSkgKwogIGxhYnMoeCA9ICJNZXRob2QiLAogICAgICAgeSA9ICJBYnNvbHV0ZSBSVCBwcmVkaWN0aW9uIGVycm9yIChpbiBzKSIsCiAgICAgICBjb2xvdXIgPSAiQ291cnNlIikKYGBgCgpGaXQgYSByZWdyZXNzaW9uIG1vZGVsIG9uIGFic29sdXRlIFJUIHByZWRpY3Rpb24gZXJyb3IuCgoKIyMjIyMgRnJlbmNoCgpgYGB7cn0KbV9ydF9wcmVkX2Vycm9yX2dsX2ZpbGUgPC0gZmlsZS5wYXRoKCIuLiIsICJkYXRhIiwgIm1vZGVsX2ZpdHMiLCAibV9ydF9wcmVkX2Vycm9yX0dyYW5kZXNfTGlnbmVzLnJkYSIpCgppZiAoZmlsZS5leGlzdHMobV9ydF9wcmVkX2Vycm9yX2dsX2ZpbGUpKSB7CiAgbG9hZChtX3J0X3ByZWRfZXJyb3JfZ2xfZmlsZSkKfSBlbHNlIHsKICAKICBwcmVkX2dsX3JlZyA8LSAoCiAgICBwcmVkX3J0X2Vycm9yCiAgICBbY291cnNlID09ICJGcmVuY2giXQogICAgW3NhbXBsZSguTiwgMWU2KV0KICAgIFssIC4ocHJlZGljdGlvbl9sYWJlbCwgYWJzX3J0X3ByZWRfZXJyb3IsIHVzZXJfaWQsIGZhY3RfaWQpXQogICkKICAKICBtX3J0X3ByZWRfZXJyb3JfZ2wgPC0gbG1lcihhYnNfcnRfcHJlZF9lcnJvciB+IHByZWRpY3Rpb25fbGFiZWwgKyAKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICgxIHwgdXNlcl9pZCkgKyAoMSB8IGZhY3RfaWQpLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgIGRhdGEgPSBwcmVkX2dsX3JlZywKICAgICAgICAgICAgICAgICAgICAgICAgICAgICBjb250cm9sID0gbG1lckNvbnRyb2wob3B0aW1pemVyID0iYm9ieXFhIikpCiAgCiAgc2F2ZShtX3J0X3ByZWRfZXJyb3JfZ2wsIGZpbGUgPSBtX3J0X3ByZWRfZXJyb3JfZ2xfZmlsZSkKfQoKc3VtbWFyeShtX3J0X3ByZWRfZXJyb3JfZ2wpCmBgYAoKQ29tcGFyZSBkaWZmZXJlbnQgcHJlZGljdGlvbiB0eXBlcyB0byBlYWNoIG90aGVyOgpgYGB7cn0KaHRfcnRfZ2wgPC0gZ2xodChtX3J0X3ByZWRfZXJyb3JfZ2wsIGxpbmZjdCA9IG1jcChwcmVkaWN0aW9uX2xhYmVsID0gIlR1a2V5IikpCnN1bW1hcnkoaHRfcnRfZ2wpCmBgYAoKSW5zcGVjdCB0aGUgbW9kZWwncyByZXNpZHVhbHM6CmBgYHtyfQpxcW5vcm0ocmVzaWQobV9ydF9wcmVkX2Vycm9yX2dsKSkKcXFsaW5lKHJlc2lkKG1fcnRfcHJlZF9lcnJvcl9nbCksIGNvbCA9ICJyZWQiKQpgYGAKCmBgYHtyfQpwbG90KG1fcnRfcHJlZF9lcnJvcl9nbCkKYGBgCgoKIyMjIyMgRW5nbGlzaAoKYGBge3J9Cm1fcnRfcHJlZF9lcnJvcl9zc19maWxlIDwtIGZpbGUucGF0aCgiLi4iLCAiZGF0YSIsICJtb2RlbF9maXRzIiwgIm1fcnRfcHJlZF9lcnJvcl9TdGVwcGluZ19TdG9uZXMucmRhIikKCmlmIChmaWxlLmV4aXN0cyhtX3J0X3ByZWRfZXJyb3Jfc3NfZmlsZSkpIHsKICBsb2FkKG1fcnRfcHJlZF9lcnJvcl9zc19maWxlKQp9IGVsc2UgewogIAogIHByZWRfc3NfcmVnIDwtICgKICAgIHByZWRfcnRfZXJyb3IKICAgIFtjb3Vyc2UgPT0gIkVuZ2xpc2giXQogICAgW3NhbXBsZSguTiwgMWU2KV0KICAgIFssIC4ocHJlZGljdGlvbl9sYWJlbCwgYWJzX3J0X3ByZWRfZXJyb3IsIHVzZXJfaWQsIGZhY3RfaWQpXQogICkKICAKICBtX3J0X3ByZWRfZXJyb3Jfc3MgPC0gbG1lcihhYnNfcnRfcHJlZF9lcnJvciB+IHByZWRpY3Rpb25fbGFiZWwgKyAKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICgxIHwgdXNlcl9pZCkgKyAoMSB8IGZhY3RfaWQpLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgIGRhdGEgPSBwcmVkX3NzX3JlZywKICAgICAgICAgICAgICAgICAgICAgICAgICAgICBjb250cm9sID0gbG1lckNvbnRyb2wob3B0aW1pemVyID0iYm9ieXFhIikKICApCiAgCiAgc2F2ZShtX3J0X3ByZWRfZXJyb3Jfc3MsIGZpbGUgPSBtX3J0X3ByZWRfZXJyb3Jfc3NfZmlsZSkKfQoKc3VtbWFyeShtX3J0X3ByZWRfZXJyb3Jfc3MpCmBgYAoKQ29tcGFyZSBkaWZmZXJlbnQgcHJlZGljdGlvbiB0eXBlcyB0byBlYWNoIG90aGVyOgpgYGB7cn0KaHRfcnRfc3MgPC0gZ2xodChtX3J0X3ByZWRfZXJyb3Jfc3MsIGxpbmZjdCA9IG1jcChwcmVkaWN0aW9uX2xhYmVsID0gIlR1a2V5IikpCnN1bW1hcnkoaHRfcnRfc3MpCmBgYAoKSW5zcGVjdCB0aGUgbW9kZWwncyByZXNpZHVhbHM6CmBgYHtyfQpxcW5vcm0ocmVzaWQobV9ydF9wcmVkX2Vycm9yX3NzKSkKcXFsaW5lKHJlc2lkKG1fcnRfcHJlZF9lcnJvcl9zcyksIGNvbCA9ICJyZWQiKQpgYGAKCmBgYHtyfQpwbG90KG1fcnRfcHJlZF9lcnJvcl9zcykKYGBgCgoKIyMjIyMgQ29tcGFyaXNvbgoKYGBge3J9Cmh0X3J0X2dsX3RpZHkgPC0gYnJvb206OnRpZHkoY29uZmludChodF9ydF9nbCkpCmh0X3J0X3NzX3RpZHkgPC0gYnJvb206OnRpZHkoY29uZmludChodF9ydF9zcykpCnNldERUKGh0X3J0X2dsX3RpZHkpCnNldERUKGh0X3J0X3NzX3RpZHkpCgpodF9ydF9ib3RoX3RpZHkgPC0gcmJpbmQoaHRfcnRfZ2xfdGlkeVssIGNvdXJzZSA6PSAiRnJlbmNoIl0sCiAgICAgICAgICAgICAgICAgICAgICBodF9ydF9zc190aWR5WywgY291cnNlIDo9ICJFbmdsaXNoIl0pCmBgYAoKYGBge3J9CnBfcnRfcHJlZF9lcnJvcl9jb21wIDwtIGdncGxvdChodF9ydF9ib3RoX3RpZHksIGFlcyh4ID0gbGhzLCB5ID0gZXN0aW1hdGUsIHltaW4gPSBjb25mLmxvdywgeW1heCA9IGNvbmYuaGlnaCwgY29sb3VyID0gY291cnNlKSkgKwogIGdlb21faGxpbmUoeWludGVyY2VwdCA9IDAsIGxpbmV0eXBlID0gIjExIiwgY29sb3VyID0gImdyZXk2MCIpICsKICBnZW9tX2Vycm9yYmFyKHdpZHRoID0gMC4xKSArIAogIGdlb21fcG9pbnQoKSArCiAgbGFicyh4ID0gIkxpbmVhciBoeXBvdGhlc2VzIiwKICAgICAgIHkgPSAiRXN0aW1hdGUiLAogICAgICAgY2FwdGlvbiA9ICJUdWtleSdzIHJhbmdlIHRlc3QuIEVycm9yIGJhcnMgc2hvdyA5NSUgZmFtaWx5LXdpc2UgY29uZmlkZW5jZSBsZXZlbC4iLAogICAgICAgY29sb3VyID0gIkNvdXJzZSIpICsKICBjb29yZF9mbGlwKCkKCnBfcnRfcHJlZF9lcnJvcl9jb21wCgpnZ3NhdmUocGxvdCA9IHBfcnRfcHJlZF9lcnJvcl9jb21wLCBmaWxlLnBhdGgoIi4uIiwgIm91dHB1dCIsICJydF9wcmVkaWN0aW9uX2Vycm9yX2NvbXBhcmlzb25zLnBuZyIpLAogICAgICAgZGV2aWNlID0gInBuZyIsIHdpZHRoID0gNy41LCBoZWlnaHQgPSA1KQoKcm0ocF9ydF9wcmVkX2Vycm9yX2NvbXApCmBgYAoKCiMjIyMjIFN1bW1hcnkgcGxvdAoKYGBge3J9CnByZWRfcnRfZXJyb3JfYXZnWywgcHJlZGljdGlvbl9yYW5rIDo9IGZyYW5rKC1tYWUpLCBieSA9IC4oY291cnNlKV0KCmFubm90YXRpb25fZGZfc3MgPC0gZGF0YS50YWJsZSgKICBjb3Vyc2UgPSByZXAoIkVuZ2xpc2giLCAxMCksCiAgc3RhcnQgPSBjKDEsIDEsIDEsIDEsCiAgICAgICAgICAgIDIsIDIsIDIsCiAgICAgICAgICAgIDMsIDMsCiAgICAgICAgICAgIDQKICApLAogIGVuZCA9IGMoMiwgMywgNCwgNSwKICAgICAgICAgIDMsIDQsIDUsCiAgICAgICAgICA0LCA1LAogICAgICAgICAgNQogICksCiAgeSA9IHNlcShtYXgocHJlZF9ydF9lcnJvcl9hdmckbWFlKSoxLjAxICsgNDUsIG1heChwcmVkX3J0X2Vycm9yX2F2ZyRtYWUpKjEuMDEsIGJ5ID0gLTUpLAogIGxhYmVsID0gYygicCA8IC4wMDEiLCAicCA8IC4wMDEiLCAicCA8IC4wMDEiLCAicCA8IC4wMDEiLAogICAgICAgICAgICAibi5zLiIsICJwIDwgLjAwMSIsICJwIDwgLjAwMSIsCiAgICAgICAgICAgICJuLnMuIiwgInAgPCAuMDEiLAogICAgICAgICAgICAibi5zLiIpCikKCmFubm90YXRpb25fZGZfZ2wgPC0gZGF0YS50YWJsZSgKICBjb3Vyc2UgPSByZXAoIkZyZW5jaCIsIDEwKSwKICBzdGFydCA9IGMoMSwgMSwgMSwgMSwKICAgICAgICAgICAgMiwgMiwgMiwKICAgICAgICAgICAgMywgMywKICAgICAgICAgICAgNAogICksCiAgZW5kID0gYygyLCAzLCA0LCA1LAogICAgICAgICAgMywgNCwgNSwKICAgICAgICAgIDQsIDUsCiAgICAgICAgICA1CiAgKSwKICB5ID0gc2VxKG1heChwcmVkX3J0X2Vycm9yX2F2ZyRtYWUpKjEuMDEgKyA0NSwgbWF4KHByZWRfcnRfZXJyb3JfYXZnJG1hZSkqMS4wMSwgYnkgPSAtNSksCiAgbGFiZWwgPSBjKCJwIDwgLjA1IiwgInAgPCAuMDAxIiwgInAgPCAuMDAxIiwgInAgPCAuMDAxIiwKICAgICAgICAgICAgInAgPCAuMDEiLCAicCA8IC4wMDEiLCAicCA8IC4wMDEiLAogICAgICAgICAgICAicCA8IC4wMDEiLCAicCA8IC4wMDEiLAogICAgICAgICAgICAibi5zLiIpCikKCmFubm90YXRpb25fZGZfcnQgPC0gcmJpbmQoYW5ub3RhdGlvbl9kZl9zcywgYW5ub3RhdGlvbl9kZl9nbCkKYW5ub3RhdGlvbl9kZl9ydFssIGxhYmVsIDo9IGZhY3RvcihsYWJlbCwgbGV2ZWxzID0gYygicCA8IC4wMDEiLCAicCA8IC4wMSIsICJwIDwgLjA1IiwgIm4ucy4iKSldCgpwX3J0X3ByZWRfZXJyb3Jfc3VtbWFyeSA8LSBnZ3Bsb3QocHJlZF9ydF9lcnJvcl9hdmcsIGFlcyh4ID0gcHJlZGljdGlvbl9yYW5rLCB5ID0gbWFlKSkgKwogIGZhY2V0X2dyaWQofiBjb3Vyc2UpICsKICBnZW9tX2xpbmUoZGF0YSA9IGFubm90YXRpb25fZGZfcnQsCiAgICAgICAgICAgIGFlcyh4ID0gMSwgeSA9IDEzMDAsIGx0eSA9IGxhYmVsLCBhbHBoYSA9IGxhYmVsLCBjb2xvdXIgPSBOVUxMKSkgKyAjIER1bW15IGxpbmUgdG8gZ2V0IGxlZ2VuZAogIGdlb21fbGluZShhZXMoY29sb3VyID0gY291cnNlLCBncm91cCA9IGNvdXJzZSkpICsKICBnZW9tX2Vycm9yYmFyKGFlcyh5bWluID0gbWFlIC0gYWVfc2UsIHltYXggPSBtYWUgKyBhZV9zZSksIHdpZHRoID0gMCkgKwogIGdlb21fcG9pbnQoYWVzKGNvbG91ciA9IGNvdXJzZSwgZ3JvdXAgPSBjb3Vyc2UpKSArCiAgZ2VvbV9sYWJlbChhZXMobGFiZWwgPSBwcmVkaWN0aW9uX2xhYmVsKSwgCiAgICAgICAgICAgICBjb2xvdXIgPSAiYmxhY2siLCAKICAgICAgICAgICAgIGFscGhhID0gLjksCiAgICAgICAgICAgICBsYWJlbC5zaXplID0gTkEsIAogICAgICAgICAgICAgbnVkZ2VfeSA9IC0xNSkgKwogIGxhYnMoeCA9IE5VTEwsCiAgICAgICB5ID0gIkFic29sdXRlIHByZWRpY3Rpb24gZXJyb3I6XG5yZXNwb25zZSB0aW1lIChzKSIsCiAgICAgICBjb2xvdXIgPSAiQ291cnNlIikgKwogIHNjYWxlX3hfY29udGludW91cyhleHBhbmQgPSBleHBhbnNpb24oYWRkID0gLjc1KSwgYnJlYWtzID0gTlVMTCkgKwogIHNjYWxlX3lfY29udGludW91cyhsYWJlbHMgPSBzY2FsZXM6OmNvbW1hX2Zvcm1hdChiaWcubWFyayA9ICIuIikpICsKICBzY2FsZV9jb2xvdXJfbWFudWFsKHZhbHVlcyA9IGRhdGFzZXRfY29sb3VycykgKwogIHNjYWxlX2xpbmV0eXBlX21hbnVhbCh2YWx1ZXMgPSBjKCJwIDwgLjAwMSIgPSAxLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICJwIDwgLjAxIiA9IDUsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgInAgPCAuMDUiID0gMiwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAibi5zLiIgPSAzKSwKICAgICAgICAgICAgICAgICAgICAgICAgbmFtZSA9ICJQYWlyd2lzZSBjb21wYXJpc29uOiIpICsKICBzY2FsZV9hbHBoYV9tYW51YWwodmFsdWVzID0gYygicCA8IC4wMDEiID0gMSwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAicCA8IC4wMSIgPSAuNzUsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgInAgPCAuMDUiID0gLjUsIAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICJuLnMuIiA9IC4yNSksIAogICAgICAgICAgICAgICAgICAgICBuYW1lID0gIlBhaXJ3aXNlIGNvbXBhcmlzb246IikgKwogIGd1aWRlcyhjb2xvdXIgPSAibm9uZSIpICsKICBnZ3NpZ25pZjo6Z2VvbV9zaWduaWYoZGF0YSA9IGFubm90YXRpb25fZGZfcnQsCiAgICAgICAgICAgICAgICAgICAgICAgIGFlcyh4bWluID0gc3RhcnQsIHhtYXggPSBlbmQsIGFubm90YXRpb25zID0gIiIsIAogICAgICAgICAgICAgICAgICAgICAgICAgICAgeV9wb3NpdGlvbiA9IHksIGx0eSA9IGxhYmVsLCBhbHBoYSA9IGxhYmVsKSwKICAgICAgICAgICAgICAgICAgICAgICAgdGlwX2xlbmd0aCA9IDAsCiAgICAgICAgICAgICAgICAgICAgICAgIG1hbnVhbCA9IFRSVUUpICArCiAgdGhlbWUobGVnZW5kLnBvc2l0aW9uID0gImJvdHRvbSIsCiAgICAgICAgbGVnZW5kLmp1c3RpZmljYXRpb24gPSAicmlnaHQiKQoKcF9ydF9wcmVkX2Vycm9yX3N1bW1hcnkKCmdnc2F2ZShmaWxlLnBhdGgoIi4uIiwgIm91dHB1dCIsICJydF9hYnNvbHV0ZV9wcmVkaWN0aW9uX2Vycm9yX3N1bW1hcnkucG5nIiksCiAgICAgICBkZXZpY2UgPSAicG5nIiwgd2lkdGggPSAxMCwgaGVpZ2h0ID0gNCkKYGBgCgojIyMjIyBJbXByb3ZlbWVudAoKSG93IGJpZyB3YXMgdGhlIGltcHJvdmVtZW50IGZyb20gd29yc3QgdG8gYmVzdCBwcmVkaWN0aW9uIG1ldGhvZD8KCkZyZW5jaDoKYGBge3J9CiMgQWJzb2x1dGUgY2hhbmdlCmh0X3J0X2dsX3RpZHlbbGhzID09ICJGYWN0IC0gRGVmYXVsdCIsIGVzdGltYXRlWzFdXQoKIyAlIGNoYW5nZQpzY2FsZXM6OnBlcmNlbnQoCiAgaHRfcnRfZ2xfdGlkeVtsaHMgPT0gIkZhY3QgLSBEZWZhdWx0IiwgZXN0aW1hdGVbMV1dIC8gZml4ZWYobV9ydF9wcmVkX2Vycm9yX2dsKVtbMV1dLAogIGFjY3VyYWN5ID0gLjEpCmBgYAoKRW5nbGlzaDoKYGBge3J9CiMgQWJzb2x1dGUgY2hhbmdlCmh0X3J0X3NzX3RpZHlbbGhzID09ICJGYWN0ICYgTGVhcm5lciAtIERlZmF1bHQiLCBlc3RpbWF0ZVsxXV0KCiMgJSBjaGFuZ2UKc2NhbGVzOjpwZXJjZW50KAogIGh0X3J0X3NzX3RpZHlbbGhzID09ICJGYWN0ICYgTGVhcm5lciAtIERlZmF1bHQiLCBlc3RpbWF0ZVsxXV0gLyBmaXhlZihtX3J0X3ByZWRfZXJyb3Jfc3MpW1sxXV0sCiAgYWNjdXJhY3kgPSAuMSkKYGBgCgojIFJlc3BvbnNlIGFjY3VyYWN5CgojIyBQcmVkaWN0ZWQgcmVzcG9uc2UgYWNjdXJhY3kKCiMjIyBEaXN0cmlidXRpb24gb2YgcHJlZGljdGlvbnMKCmBgYHtyfQpwX2FjY19wcmVkX2Rpc3QgPC0gZ2dwbG90KHByZWRfYmVoLCBhZXMoeCA9IHByZWRpY3RlZF9hY2N1cmFjeSwgZmlsbCA9IHByZWRpY3Rpb25fbGFiZWwpKSArCiAgICBmYWNldF9ncmlkKGNvdXJzZSB+IHByZWRpY3Rpb25fbGFiZWwsIHNjYWxlcyA9ICJmcmVlX3kiKSArCiAgICBnZW9tX2hpc3RvZ3JhbShiaW53aWR0aCA9IC4wMSkgKwogICAgZ3VpZGVzKGZpbGwgPSAibm9uZSIpICsKICAgIGxhYnMoeCA9ICJQcmVkaWN0ZWQgYWNjdXJhY3kiKSArCiAgICBzY2FsZV94X2NvbnRpbnVvdXMoYnJlYWtzID0gc2VxKDAsIDEsIGJ5ID0gLjI1KSwgbGFiZWxzID0gc2NhbGVzOjpwZXJjZW50X2Zvcm1hdCgpKSArCiAgICBzY2FsZV9maWxsX21hbnVhbCh2YWx1ZXMgPSBjb25kaXRpb25fY29sb3VycykKICAKcF9hY2NfcHJlZF9kaXN0CgpnZ3NhdmUocGxvdCA9IHBfYWNjX3ByZWRfZGlzdCwgZmlsZS5wYXRoKCIuLiIsICJvdXRwdXQiLCAiYWNjX3ByZWRpY3RlZF9kaXN0cmlidXRpb24ucG5nIiksCiAgICAgICAgIGRldmljZSA9ICJwbmciLCB3aWR0aCA9IDYsIGhlaWdodCA9IDcuNSkKCnJtKHBfYWNjX3ByZWRfZGlzdCkKYGBgCgoKIyMjIFByZWRpY3RlZCB2cyBvYnNlcnZlZCB2YWx1ZXMKCmBgYHtyfQpwbG90X2RvZGdlIDwtIGZ1bmN0aW9uKHksIGRvZGdlID0gLjEpIHsKICByZXR1cm4gKHkgKiAoMSArIGRvZGdlKSAtIGRvZGdlLzIpCn0KYGBgCgoKYGBge3J9CnBfYWNjX3ByZWRfdl9vYnMgPC0gZ2dwbG90KHByZWRfYmVoLCBhZXMoeCA9IHByZWRpY3RlZF9hY2N1cmFjeSwgeSA9IGNvcnJlY3QsIGdyb3VwID0gcHJlZGljdGlvbl9sYWJlbCwgY29sb3VyID0gcHJlZGljdGlvbl9sYWJlbCwgZmlsbCA9IHByZWRpY3Rpb25fbGFiZWwpKSArCiAgICBmYWNldF9ncmlkKGNvdXJzZSB+IHByZWRpY3Rpb25fbGFiZWwpICsKICAgIGdlb21fcG9pbnQoYWVzKHkgPSBjb3JyZWN0KSwKICAgICAgICAgICAgICAgcG9zaXRpb24gPSBwb3NpdGlvbl9qaXR0ZXIod2lkdGggPSAwLCBoZWlnaHQgPSAuMDI1LCBzZWVkID0gMTIzKSwKICAgICAgICAgICAgICAgc2l6ZSA9IC4wMDEsIHBjaCA9ICIuIiwgYWxwaGEgPSAuMSkgKwogICAgbGFicyh4ID0gIlByZWRpY3RlZCBhY2N1cmFjeSIsCiAgICAgICAgIHkgPSAiUmVzcG9uc2UgYWNjdXJhY3kiLAogICAgICAgICBjb2xvdXIgPSAiUHJlZGljdGlvbiBtZXRob2QiLAogICAgICAgICBmaWxsID0gIlByZWRpY3Rpb24gbWV0aG9kIikgKwogIGd1aWRlcyhjb2xvdXIgPSAibm9uZSIsCiAgICAgICAgIGZpbGwgPSAibm9uZSIpICsKICAgIHNjYWxlX3hfY29udGludW91cyhicmVha3MgPSBzZXEoMCwgMSwgYnkgPSAuMjUpLCBsYWJlbHMgPSBzY2FsZXM6OnBlcmNlbnRfZm9ybWF0KCkpICsKICAgIHNjYWxlX3lfY29udGludW91cyhicmVha3MgPSBzZXEoMCwgMSwgYnkgPSAuMjUpLCBsYWJlbHMgPSBzY2FsZXM6OnBlcmNlbnRfZm9ybWF0KCkpICsKICAgIHNjYWxlX2NvbG91cl9tYW51YWwodmFsdWVzID0gY29uZGl0aW9uX2NvbG91cnMpICsKICAgIHNjYWxlX2ZpbGxfbWFudWFsKHZhbHVlcyA9IGNvbmRpdGlvbl9jb2xvdXJzKSArCiAgICBjb29yZF9jYXJ0ZXNpYW4oeGxpbSA9IGMoMCwgMSksIHlsaW0gPSBjKDAsIDEpLCBjbGlwID0gIm9mZiIpCgpwX2FjY19wcmVkX3Zfb2JzCgpnZ3NhdmUocGxvdCA9IHBfYWNjX3ByZWRfdl9vYnMsIGZpbGUucGF0aCgiLi4iLCAib3V0cHV0IiwgImFjY19wcmVkaWN0ZWRfdnNfb2JzZXJ2ZWQucG5nIiksCiAgICAgICBkZXZpY2UgPSAicG5nIiwgd2lkdGggPSAxMCwgaGVpZ2h0ID0gNC41KQoKcm0ocF9hY2NfcHJlZF92X29icykKYGBgCgojIyMjIFByZWRpY3Rpb24gZXJyb3IKCmBgYHtyfQpwcmVkX2FjY19lcnJvciA8LSAocHJlZF9iZWgKICAgICAgICAgICAgICAgICAgIFssIGFjY19wcmVkX2Vycm9yIDo9IHByZWRpY3RlZF9hY2N1cmFjeSAtIGNvcnJlY3RdCiAgICAgICAgICAgICAgICAgICBbLCBhYnNfYWNjX3ByZWRfZXJyb3IgOj0gYWJzKGFjY19wcmVkX2Vycm9yKV0pCgoKcHJlZF9hY2NfZXJyb3JfYXZnIDwtIHByZWRfYWNjX2Vycm9yWywgLihtYWUgPSBtZWFuKGFic19hY2NfcHJlZF9lcnJvciksCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgYWVfc2UgPSBzZChhYnNfYWNjX3ByZWRfZXJyb3IpLy5OKSwgCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBieSA9IC4oY291cnNlLCBwcmVkaWN0aW9uX2xhYmVsKV0KCm5fb2JzIDwtIHByZWRfYWNjX2Vycm9yWywgLk4sIGJ5ID0gLihjb3Vyc2UsIHByZWRpY3Rpb25fbGFiZWwpXQpgYGAKCkRpc3RyaWJ1dGlvbiBvZiBwcmVkaWN0aW9uIGVycm9yOgpgYGB7cn0KcF9hY2NfcHJlZF9lcnJvciA8LSBnZ3Bsb3QocHJlZF9hY2NfZXJyb3IsIGFlcyh4ID0gYWNjX3ByZWRfZXJyb3IsIGZpbGwgPSBwcmVkaWN0aW9uX2xhYmVsKSkgKwogIGZhY2V0X2dyaWQocHJlZGljdGlvbl9sYWJlbCB+IGNvdXJzZSAsIHNjYWxlcyA9ICJmcmVlX3kiKSArCiAgZ2VvbV9oaXN0b2dyYW0oYWVzKHkgPSAuLmRlbnNpdHkuLiksIGJpbndpZHRoID0gLjAxKSArCiAgZ3VpZGVzKGZpbGwgPSAibm9uZSIpICsKICBsYWJzKHggPSAiQWNjdXJhY3kgcHJlZGljdGlvbiBlcnJvciAocHJlZGljdGVkIC0gb2JzZXJ2ZWQpIiwKICAgICAgIHkgPSAiRGVuc2l0eSIpICsKICBjb29yZF9jYXJ0ZXNpYW4oeGxpbSA9IGMoLTEsIDEpKSArCiAgc2NhbGVfZmlsbF9tYW51YWwodmFsdWVzID0gY29uZGl0aW9uX2NvbG91cnMpCgpwX2FjY19wcmVkX2Vycm9yCgpnZ3NhdmUocGxvdCA9IHBfYWNjX3ByZWRfZXJyb3IsIGZpbGUucGF0aCgiLi4iLCAib3V0cHV0IiwgImFjY19wcmVkaWN0aW9uX2Vycm9yLnBuZyIpLAogICAgICAgZGV2aWNlID0gInBuZyIsIHdpZHRoID0gNSwgaGVpZ2h0ID0gNy41KQoKcm0ocF9hY2NfcHJlZF9lcnJvcikKYGBgCgojIyMjIEFic29sdXRlIHByZWRpY3Rpb24gZXJyb3IKCmBgYHtyfQpwX2Fic19hY2NfcHJlZF9lcnJvciA8LSBnZ3Bsb3QocHJlZF9hY2NfZXJyb3IsIGFlcyh4ID0gYWJzX2FjY19wcmVkX2Vycm9yLCBmaWxsID0gcHJlZGljdGlvbl9sYWJlbCkpICsKICBmYWNldF9ncmlkKHByZWRpY3Rpb25fbGFiZWwgfiBjb3Vyc2UgLCBzY2FsZXMgPSAiZnJlZV95IikgKwogIGdlb21faGlzdG9ncmFtKGFlcyh5ID0gLi5kZW5zaXR5Li4pLCBiaW53aWR0aCA9IC4wMSkgKwogIGd1aWRlcyhmaWxsID0gIm5vbmUiKSArCiAgbGFicyh4ID0gIkFjY3VyYWN5IHByZWRpY3Rpb24gZXJyb3IgKHByZWRpY3RlZCAtIG9ic2VydmVkKSIsCiAgICAgICB5ID0gIkRlbnNpdHkiKSArCiAgY29vcmRfY2FydGVzaWFuKHhsaW0gPSBjKDAsIDEpKSArCiAgc2NhbGVfZmlsbF9tYW51YWwodmFsdWVzID0gY29uZGl0aW9uX2NvbG91cnMpCgpwX2Fic19hY2NfcHJlZF9lcnJvcgoKZ2dzYXZlKHBsb3QgPSBwX2Fic19hY2NfcHJlZF9lcnJvciwgZmlsZS5wYXRoKCIuLiIsICJvdXRwdXQiLCAiYWNjX2Fic29sdXRlX3ByZWRpY3Rpb25fZXJyb3IucG5nIiksCiAgICAgICBkZXZpY2UgPSAicG5nIiwgd2lkdGggPSA1LCBoZWlnaHQgPSA3LjUpCgpybShwX2Fic19hY2NfcHJlZF9lcnJvcikKYGBgCgpgYGB7cn0KZ2dwbG90KHByZWRfYWNjX2Vycm9yX2F2ZywgYWVzKHggPSBwcmVkaWN0aW9uX2xhYmVsLCB5ID0gbWFlLCBjb2xvdXIgPSBjb3Vyc2UpKSArCiAgZ2VvbV9ib3hwbG90KGRhdGEgPSBwcmVkX2FjY19lcnJvciwKICAgICAgICAgICAgICAgYWVzKHkgPSBhYnNfYWNjX3ByZWRfZXJyb3IsIGdyb3VwID0gaW50ZXJhY3Rpb24oY291cnNlLCBwcmVkaWN0aW9uX2xhYmVsKSksCiAgICAgICAgICAgICAgIGNvbG91ciA9ICJncmV5NzAiLAogICAgICAgICAgICAgICB3aWR0aCA9IC4yNSwKICAgICAgICAgICAgICAgb3V0bGllci5zaGFwZSA9IE5BLAogICAgICAgICAgICAgICBwb3NpdGlvbiA9IHBvc2l0aW9uX2RvZGdlKHdpZHRoID0gLjUpKSArCiAgZ2VvbV9lcnJvcmJhcihhZXMoeW1pbiA9IG1hZSAtIGFlX3NlLCB5bWF4ID0gbWFlICsgYWVfc2UpLCB3aWR0aCA9IDAsIHBvc2l0aW9uID0gcG9zaXRpb25fZG9kZ2Uod2lkdGggPSAuNSkpICsKICBnZW9tX3BvaW50KHBvc2l0aW9uID0gcG9zaXRpb25fZG9kZ2Uod2lkdGggPSAuNSkpICsKICBjb29yZF9jYXJ0ZXNpYW4oeWxpbSA9IGMoMCwgMSkpICsKICBsYWJzKHggPSAiTWV0aG9kIiwKICAgICAgIHkgPSAiQWJzb2x1dGUgYWNjdXJhY3kgcHJlZGljdGlvbiBlcnJvciIsCiAgICAgICBjb2xvdXIgPSAiQ291cnNlIikKYGBgCgpGaXQgYSByZWdyZXNzaW9uIG1vZGVsLgoKIyMjIyMgRnJlbmNoCgpgYGB7cn0KbV9hY2NfcHJlZF9lcnJvcl9nbF9maWxlIDwtIGZpbGUucGF0aCgiLi4iLCAiZGF0YSIsICJtb2RlbF9maXRzIiwgIm1fYWNjX3ByZWRfZXJyb3JfR3JhbmRlc19MaWduZXMucmRhIikKCmlmIChmaWxlLmV4aXN0cyhtX2FjY19wcmVkX2Vycm9yX2dsX2ZpbGUpKSB7CiAgbG9hZChtX2FjY19wcmVkX2Vycm9yX2dsX2ZpbGUpCn0gZWxzZSB7CiAgCiAgcHJlZF9nbF9yZWcgPC0gKAogICAgcHJlZF9hY2NfZXJyb3IKICAgIFtjb3Vyc2UgPT0gIkZyZW5jaCJdCiAgICBbc2FtcGxlKC5OLCAxZTYpXQogICAgWywgLihwcmVkaWN0aW9uX2xhYmVsLCBhYnNfYWNjX3ByZWRfZXJyb3IsIHVzZXJfaWQsIGZhY3RfaWQpXQogICkKICAKICBtX2FjY19wcmVkX2Vycm9yX2dsIDwtIGxtZXIoYWJzX2FjY19wcmVkX2Vycm9yIH4gcHJlZGljdGlvbl9sYWJlbCArIAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgKDEgfCB1c2VyX2lkKSArICgxIHwgZmFjdF9pZCksCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgZGF0YSA9IHByZWRfZ2xfcmVnLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgIGNvbnRyb2wgPSBsbWVyQ29udHJvbChvcHRpbWl6ZXIgPSJib2J5cWEiKSkKICAgIAogIHNhdmUobV9hY2NfcHJlZF9lcnJvcl9nbCwgZmlsZSA9IG1fYWNjX3ByZWRfZXJyb3JfZ2xfZmlsZSkKfQoKc3VtbWFyeShtX2FjY19wcmVkX2Vycm9yX2dsKQpgYGAKCkNvbXBhcmUgZGlmZmVyZW50IHByZWRpY3Rpb24gdHlwZXMgdG8gZWFjaCBvdGhlcjoKYGBge3J9Cmh0X2FjY19nbCA8LSBnbGh0KG1fYWNjX3ByZWRfZXJyb3JfZ2wsIGxpbmZjdCA9IG1jcChwcmVkaWN0aW9uX2xhYmVsID0gIlR1a2V5IikpCnN1bW1hcnkoaHRfYWNjX2dsKQpgYGAKCkluc3BlY3QgdGhlIG1vZGVsJ3MgcmVzaWR1YWxzOgpgYGB7cn0KcXFub3JtKHJlc2lkKG1fYWNjX3ByZWRfZXJyb3JfZ2wpKQpxcWxpbmUocmVzaWQobV9hY2NfcHJlZF9lcnJvcl9nbCksIGNvbCA9ICJyZWQiKQpgYGAKCmBgYHtyfQpwbG90KG1fYWNjX3ByZWRfZXJyb3JfZ2wpCmBgYAoKIyMjIyMgRW5nbGlzaAoKYGBge3J9Cm1fYWNjX3ByZWRfZXJyb3Jfc3NfZmlsZSA8LSBmaWxlLnBhdGgoIi4uIiwgImRhdGEiLCAibW9kZWxfZml0cyIsICJtX2FjY19wcmVkX2Vycm9yX1N0ZXBwaW5nX1N0b25lcy5yZGEiKQoKaWYgKGZpbGUuZXhpc3RzKG1fYWNjX3ByZWRfZXJyb3Jfc3NfZmlsZSkpIHsKICBsb2FkKG1fYWNjX3ByZWRfZXJyb3Jfc3NfZmlsZSkKfSBlbHNlIHsKICAKICBwcmVkX3NzX3JlZyA8LSAoCiAgICBwcmVkX2FjY19lcnJvcgogICAgW2NvdXJzZSA9PSAiRW5nbGlzaCJdCiAgICBbc2FtcGxlKC5OLCAxZTYpXQogICAgWywgLihwcmVkaWN0aW9uX2xhYmVsLCBhYnNfYWNjX3ByZWRfZXJyb3IsIHVzZXJfaWQsIGZhY3RfaWQpXQogICkKICAKICBtX2FjY19wcmVkX2Vycm9yX3NzIDwtIGxtZXIoYWJzX2FjY19wcmVkX2Vycm9yIH4gcHJlZGljdGlvbl9sYWJlbCArIAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgKDEgfCB1c2VyX2lkKSArICgxIHwgZmFjdF9pZCksCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgZGF0YSA9IHByZWRfc3NfcmVnLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgIGNvbnRyb2wgPSBsbWVyQ29udHJvbChvcHRpbWl6ZXIgPSJib2J5cWEiKQogICkKICAKICBzYXZlKG1fYWNjX3ByZWRfZXJyb3Jfc3MsIGZpbGUgPSBtX2FjY19wcmVkX2Vycm9yX3NzX2ZpbGUpCn0KCnN1bW1hcnkobV9hY2NfcHJlZF9lcnJvcl9zcykKYGBgCgpDb21wYXJlIGRpZmZlcmVudCBwcmVkaWN0aW9uIHR5cGVzIHRvIGVhY2ggb3RoZXI6CmBgYHtyfQpodF9hY2Nfc3MgPC0gZ2xodChtX2FjY19wcmVkX2Vycm9yX3NzLCBsaW5mY3QgPSBtY3AocHJlZGljdGlvbl9sYWJlbCA9ICJUdWtleSIpKQpzdW1tYXJ5KGh0X2FjY19zcykKYGBgCgpJbnNwZWN0IHRoZSBtb2RlbCdzIHJlc2lkdWFsczoKYGBge3J9CnFxbm9ybShyZXNpZChtX2FjY19wcmVkX2Vycm9yX3NzKSkKcXFsaW5lKHJlc2lkKG1fYWNjX3ByZWRfZXJyb3Jfc3MpLCBjb2wgPSAicmVkIikKYGBgCgpgYGB7cn0KcGxvdChtX2FjY19wcmVkX2Vycm9yX3NzKQpgYGAKCiMjIyMjIENvbXBhcmlzb24KCmBgYHtyfQpodF9hY2NfZ2xfdGlkeSA8LSBicm9vbTo6dGlkeShjb25maW50KGh0X2FjY19nbCkpCmh0X2FjY19zc190aWR5IDwtIGJyb29tOjp0aWR5KGNvbmZpbnQoaHRfYWNjX3NzKSkKc2V0RFQoaHRfYWNjX2dsX3RpZHkpCnNldERUKGh0X2FjY19zc190aWR5KQoKaHRfYWNjX2JvdGhfdGlkeSA8LSByYmluZChodF9hY2NfZ2xfdGlkeVssIGNvdXJzZSA6PSAiRnJlbmNoIl0sCiAgICAgICAgICAgICAgICAgICAgICBodF9hY2Nfc3NfdGlkeVssIGNvdXJzZSA6PSAiRW5nbGlzaCJdKQpgYGAKCmBgYHtyfQpwX2FjY19wcmVkX2Vycm9yX2NvbXAgPC0gZ2dwbG90KGh0X2FjY19ib3RoX3RpZHksIGFlcyh4ID0gbGhzLCB5ID0gZXN0aW1hdGUsIHltaW4gPSBjb25mLmxvdywgeW1heCA9IGNvbmYuaGlnaCwgY29sb3VyID0gY291cnNlKSkgKwogIGdlb21faGxpbmUoeWludGVyY2VwdCA9IDAsIGxpbmV0eXBlID0gIjExIiwgY29sb3VyID0gImdyZXk2MCIpICsKICBnZW9tX2Vycm9yYmFyKHdpZHRoID0gMC4xKSArIAogIGdlb21fcG9pbnQoKSArCiAgbGFicyh4ID0gIkxpbmVhciBoeXBvdGhlc2VzIiwKICAgICAgIHkgPSAiRXN0aW1hdGUiLAogICAgICAgY2FwdGlvbiA9ICJUdWtleSdzIHJhbmdlIHRlc3QuIEVycm9yIGJhcnMgc2hvdyA5NSUgZmFtaWx5LXdpc2UgY29uZmlkZW5jZSBsZXZlbC4iLAogICAgICAgY29sb3VyID0gIkNvdXJzZSIpICsKICBjb29yZF9mbGlwKCkKCnBfYWNjX3ByZWRfZXJyb3JfY29tcAoKZ2dzYXZlKHBsb3QgPSBwX2FjY19wcmVkX2Vycm9yX2NvbXAsIGZpbGUucGF0aCgiLi4iLCAib3V0cHV0IiwgImFjY19wcmVkaWN0aW9uX2Vycm9yX2NvbXBhcmlzb25zLnBuZyIpLAogICAgICAgZGV2aWNlID0gInBuZyIsIHdpZHRoID0gNy41LCBoZWlnaHQgPSA1KQoKcm0ocF9hY2NfcHJlZF9lcnJvcl9jb21wKQpgYGAKCgojIyMjIyBTdW1tYXJ5IHBsb3QKCmBgYHtyfQpwcmVkX2FjY19lcnJvcl9hdmdbLCBwcmVkaWN0aW9uX3JhbmsgOj0gZnJhbmsoLW1hZSksIGJ5ID0gLihjb3Vyc2UpXQoKYW5ub3RhdGlvbl9kZl9zcyA8LSBkYXRhLnRhYmxlKAogIGNvdXJzZSA9IHJlcCgiRW5nbGlzaCIsIDEwKSwKICBzdGFydCA9IGMoMSwgMSwgMSwgMSwKICAgICAgICAgICAgMiwgMiwgMiwKICAgICAgICAgICAgMywgMywKICAgICAgICAgICAgNAogICksCiAgZW5kID0gYygyLCAzLCA0LCA1LAogICAgICAgICAgMywgNCwgNSwKICAgICAgICAgIDQsIDUsCiAgICAgICAgICA1CiAgKSwKICB5ID0gc2VxKG1heChwcmVkX2FjY19lcnJvcl9hdmckbWFlKSoxLjAxICsgLjAxMTI1LCBtYXgocHJlZF9hY2NfZXJyb3JfYXZnJG1hZSkqMS4wMSwgYnkgPSAtLjAwMTI1KSwKICBsYWJlbCA9IGMoInAgPCAuMDAxIiwgInAgPCAuMDAxIiwgInAgPCAuMDAxIiwgInAgPCAuMDAxIiwKICAgICAgICAgICAgInAgPCAuMDAxIiwgInAgPCAuMDAxIiwgInAgPCAuMDAxIiwKICAgICAgICAgICAgInAgPCAuMDAxIiwgInAgPCAuMDAxIiwKICAgICAgICAgICAgInAgPCAuMDAxIikKKQoKYW5ub3RhdGlvbl9kZl9nbCA8LSBkYXRhLnRhYmxlKAogIGNvdXJzZSA9IHJlcCgiRnJlbmNoIiwgMTApLAogIHN0YXJ0ID0gYygxLCAxLCAxLCAxLAogICAgICAgICAgICAyLCAyLCAyLAogICAgICAgICAgICAzLCAzLAogICAgICAgICAgICA0CiAgKSwKICBlbmQgPSBjKDIsIDMsIDQsIDUsCiAgICAgICAgICAzLCA0LCA1LAogICAgICAgICAgNCwgNSwKICAgICAgICAgIDUKICApLAogIHkgPSBzZXEobWF4KHByZWRfYWNjX2Vycm9yX2F2ZyRtYWUpKjEuMDEgKyAuMDExMjUsIG1heChwcmVkX2FjY19lcnJvcl9hdmckbWFlKSoxLjAxLCBieSA9IC0uMDAxMjUpLAogIGxhYmVsID0gYygicCA8IC4wMDEiLCAicCA8IC4wMDEiLCAicCA8IC4wMDEiLCAicCA8IC4wMDEiLAogICAgICAgICAgICAicCA8IC4wMDEiLCAicCA8IC4wMDEiLCAicCA8IC4wMDEiLAogICAgICAgICAgICAicCA8IC4wMDEiLCAicCA8IC4wMDEiLAogICAgICAgICAgICAicCA8IC4wMDEiKQopCgphbm5vdGF0aW9uX2RmX2FjYyA8LSByYmluZChhbm5vdGF0aW9uX2RmX3NzLCBhbm5vdGF0aW9uX2RmX2dsKQphbm5vdGF0aW9uX2RmX2FjY1ssIGxhYmVsIDo9IGZhY3RvcihsYWJlbCwgbGV2ZWxzID0gYygicCA8IC4wMDEiLCAicCA8IC4wMSIsICJwIDwgLjA1IiwgIm4ucy4iKSldCgpwX2FjY19wcmVkX2Vycm9yX3N1bW1hcnkgPC0gZ2dwbG90KHByZWRfYWNjX2Vycm9yX2F2ZywgYWVzKHggPSBwcmVkaWN0aW9uX3JhbmssIHkgPSBtYWUpKSArCiAgZmFjZXRfZ3JpZCh+IGNvdXJzZSkgKwogIGdlb21fbGluZShkYXRhID0gYW5ub3RhdGlvbl9kZl9hY2MsCiAgICAgICAgICAgIGFlcyh4ID0gMSwgeSA9IC40NSwgbHR5ID0gbGFiZWwsIGFscGhhID0gbGFiZWwsIGNvbG91ciA9IE5VTEwpKSArICMgRHVtbXkgbGluZSB0byBnZXQgbGVnZW5kCiAgZ2VvbV9saW5lKGFlcyhjb2xvdXIgPSBjb3Vyc2UsIGdyb3VwID0gY291cnNlKSkgKwogIGdlb21fZXJyb3JiYXIoYWVzKHltaW4gPSBtYWUgLSBhZV9zZSwgeW1heCA9IG1hZSArIGFlX3NlKSwgd2lkdGggPSAwKSArCiAgZ2VvbV9wb2ludChhZXMoY29sb3VyID0gY291cnNlLCBncm91cCA9IGNvdXJzZSkpICsKICBnZW9tX2xhYmVsKGFlcyhsYWJlbCA9IHByZWRpY3Rpb25fbGFiZWwpLCAKICAgICAgICAgICAgIGNvbG91ciA9ICJibGFjayIsIAogICAgICAgICAgICAgYWxwaGEgPSAuOSwKICAgICAgICAgICAgIGxhYmVsLnNpemUgPSBOQSwgCiAgICAgICAgICAgICBudWRnZV95ID0gLS4wMDQpICsKICBsYWJzKHggPSBOVUxMLAogICAgICAgeSA9ICJBYnNvbHV0ZSBwcmVkaWN0aW9uIGVycm9yOlxucmVzcG9uc2UgYWNjdXJhY3kiLAogICAgICAgY29sb3VyID0gIkNvdXJzZSIpICsKICBzY2FsZV94X2NvbnRpbnVvdXMoZXhwYW5kID0gZXhwYW5zaW9uKGFkZCA9IC43NSksIGJyZWFrcyA9IE5VTEwpICsKICBzY2FsZV9jb2xvdXJfbWFudWFsKHZhbHVlcyA9IGRhdGFzZXRfY29sb3VycykgKwogIHNjYWxlX2xpbmV0eXBlX21hbnVhbCh2YWx1ZXMgPSBjKCJwIDwgLjAwMSIgPSAxLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICJwIDwgLjAxIiA9IDUsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgInAgPCAuMDUiID0gMiwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAibi5zLiIgPSAzKSwKICAgICAgICAgICAgICAgICAgICAgICAgbmFtZSA9ICJQYWlyd2lzZSBjb21wYXJpc29uOiIpICsKICBzY2FsZV9hbHBoYV9tYW51YWwodmFsdWVzID0gYygicCA8IC4wMDEiID0gMSwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAicCA8IC4wMSIgPSAuNzUsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgInAgPCAuMDUiID0gLjUsIAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICJuLnMuIiA9IC4yNSksIAogICAgICAgICAgICAgICAgICAgICBuYW1lID0gIlBhaXJ3aXNlIGNvbXBhcmlzb246IikgKwogIGd1aWRlcyhjb2xvdXIgPSAibm9uZSIpICsKICBnZ3NpZ25pZjo6Z2VvbV9zaWduaWYoZGF0YSA9IGFubm90YXRpb25fZGZfYWNjLAogICAgICAgICAgICAgICAgICAgICAgICBhZXMoeG1pbiA9IHN0YXJ0LCB4bWF4ID0gZW5kLCBhbm5vdGF0aW9ucyA9ICIiLCAKICAgICAgICAgICAgICAgICAgICAgICAgICAgIHlfcG9zaXRpb24gPSB5LCBsdHkgPSBsYWJlbCwgYWxwaGEgPSBsYWJlbCksCiAgICAgICAgICAgICAgICAgICAgICAgIHRpcF9sZW5ndGggPSAwLAogICAgICAgICAgICAgICAgICAgICAgICBtYW51YWwgPSBUUlVFKSAgKwogIHRoZW1lKGxlZ2VuZC5wb3NpdGlvbiA9ICJib3R0b20iLAogICAgICAgIGxlZ2VuZC5qdXN0aWZpY2F0aW9uID0gInJpZ2h0IikKCnBfYWNjX3ByZWRfZXJyb3Jfc3VtbWFyeQoKZ2dzYXZlKGZpbGUucGF0aCgiLi4iLCAib3V0cHV0IiwgImFjY19hYnNvbHV0ZV9wcmVkaWN0aW9uX2Vycm9yX3N1bW1hcnkucG5nIiksCiAgICAgICBkZXZpY2UgPSAicG5nIiwgd2lkdGggPSAxMCwgaGVpZ2h0ID0gNCkKYGBgCgojIyMjIyBJbXByb3ZlbWVudAoKSG93IGJpZyB3YXMgdGhlIGltcHJvdmVtZW50IGZyb20gd29yc3QgdG8gYmVzdCBwcmVkaWN0aW9uIG1ldGhvZD8KCkZyZW5jaDoKYGBge3J9CiMgQWJzb2x1dGUgY2hhbmdlCmh0X2FjY19nbF90aWR5W2xocyA9PSAiRmFjdCAmIExlYXJuZXIgLSBEZWZhdWx0IiwgZXN0aW1hdGVbMV1dCgojICUgY2hhbmdlCnNjYWxlczo6cGVyY2VudCgKICBodF9hY2NfZ2xfdGlkeVtsaHMgPT0gIkZhY3QgJiBMZWFybmVyIC0gRGVmYXVsdCIsIGVzdGltYXRlWzFdXSAvIGZpeGVmKG1fYWNjX3ByZWRfZXJyb3JfZ2wpW1sxXV0sCiAgYWNjdXJhY3kgPSAuMSkKYGBgCgpFbmdsaXNoOgpgYGB7cn0KIyBBYnNvbHV0ZSBjaGFuZ2UKaHRfYWNjX3NzX3RpZHlbbGhzID09ICJGYWN0ICYgTGVhcm5lciAtIERlZmF1bHQiLCBlc3RpbWF0ZVsxXV0KCiMgJSBjaGFuZ2UKc2NhbGVzOjpwZXJjZW50KAogIGh0X2FjY19zc190aWR5W2xocyA9PSAiRmFjdCAmIExlYXJuZXIgLSBEZWZhdWx0IiwgZXN0aW1hdGVbMV1dIC8gZml4ZWYobV9hY2NfcHJlZF9lcnJvcl9zcylbWzFdXSwKICBhY2N1cmFjeSA9IC4xKQpgYGAKCgojIENvbWJpbmVkIHBsb3QKCmBgYHtyfQoocF9hY2NfcHJlZF9lcnJvcl9zdW1tYXJ5ICsgcF9ydF9wcmVkX2Vycm9yX3N1bW1hcnkpICsgCiAgcGxvdF9sYXlvdXQobmNvbCA9IDEsIGd1aWRlcyA9ICJjb2xsZWN0IikgKwogIHBsb3RfYW5ub3RhdGlvbih0YWdfbGV2ZWxzID0gIkEiKSAmCiAgdGhlbWUobGVnZW5kLnBvc2l0aW9uID0gImJvdHRvbSIpCgpnZ3NhdmUoZmlsZS5wYXRoKCIuLiIsICJvdXRwdXQiLCAiYmVoX2Fic29sdXRlX3ByZWRpY3Rpb25fZXJyb3Jfc3VtbWFyeS5wbmciKSwKICAgICAgIGRldmljZSA9ICJwbmciLCB3aWR0aCA9IDEwLCBoZWlnaHQgPSA4KQpgYGAKCgojIFNlc3Npb24gaW5mbwoKYGBge3J9CnNlc3Npb25JbmZvKCkKYGBg